diff --git a/compile.go b/compile.go new file mode 100644 index 0000000..b9dfbd4 --- /dev/null +++ b/compile.go @@ -0,0 +1,437 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "unicode" +) + +var useBatching bool +var compileWaitGroup sync.WaitGroup +var toCompileCount uint32 +var compileJobs []compileJob +var jobMtx sync.Mutex + +type symbolInfo struct { + name string + linePos int +} + +type labelInfo struct { + name string + num int + linePos int +} + +type compileResponse struct { + code []byte + address string +} + +type compileJob struct { + inputFile string + tempFile string + outFile string + addressExp string + response chan compileResponse +} + +func execBatchCompile(jobs []compileJob) { + const asCmdLinux string = "powerpc-eabi-as" + const objcopyCmdLinux string = "powerpc-eabi-objcopy" + + deleteFile := func(fp string) { + defer compileWaitGroup.Done() + os.Remove(fp) + } + + outputFilePath := path.Join(argConfig.ProjectRoot, "compiled.elf") + compileWaitGroup.Add(1) + defer deleteFile(outputFilePath) + + // Generate temp file names + for idx, job := range jobs { + file := job.inputFile + fileExt := filepath.Ext(file) + fileNoExt := file[0 : len(file)-len(fileExt)] + jobs[idx].tempFile = fmt.Sprintf("%s-file%d.asmtemp", fileNoExt, idx) + jobs[idx].outFile = fmt.Sprintf("%s-file%d.out", fileNoExt, idx) + } + + // Set base args + args := []string{"-a32", "-mbig", "-mregnames", "-mgekko", "-W"} + + // If defsym is defined, add it to the args + if argConfig.DefSym != "" { + args = append(args, "-defsym", argConfig.DefSym) + } + + args = append(args, "-I", argConfig.ProjectRoot) + + // Add local paths to look at when resolving includes + for _, job := range jobs { + file := job.inputFile + fileDir := filepath.Dir(file) + args = append(args, "-I", fileDir) + } + + // Set output file + args = append(args, "-o", outputFilePath) + + // Iterate through jobs, create temp files, and add them to the files to assemble + for idx, job := range jobs { + compileWaitGroup.Add(1) + defer deleteFile(job.tempFile) + + buildTempAsmFile(job.inputFile, job.addressExp, job.tempFile, fmt.Sprintf(".file%d", idx)) + args = append(args, job.tempFile) + } + + cmd := exec.Command(asCmdLinux, args...) + + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Failed to compile files") + fmt.Printf("%s", output) + panic("as failure") + } + + args = []string{outputFilePath} + for idx, job := range jobs { + compileWaitGroup.Add(1) + defer deleteFile(job.outFile) + + args = append(args, "--dump-section", fmt.Sprintf(".file%d=%s", idx, job.outFile)) + } + + cmd = exec.Command(objcopyCmdLinux, args...) + output, err = cmd.CombinedOutput() + if err != nil { + fmt.Printf("Failed to pull extract code sections\n") + fmt.Printf("%s", output) + panic("objcopy failure") + } + + for _, job := range jobs { + contents, err := ioutil.ReadFile(job.outFile) + if err != nil { + log.Panicf("Failed to read compiled file %s\n%s\n", job.outFile, err.Error()) + } + + code := contents[:len(contents)-4] + address := contents[len(contents)-4:] + if address[0] != 0x80 && address[0] != 0x81 { + log.Panicf( + "Injection address in file %s evaluated to a value that does not start with 0x80 or 0x81"+ + ", probably an invalid address\n", + job.inputFile, + ) + } + + job.response <- compileResponse{code: code, address: fmt.Sprintf("%x", address)} + } +} + +func batchCompile(file, addressExp string) ([]byte, string) { + c := make(chan compileResponse) + jobMtx.Lock() + compileJobs = append(compileJobs, compileJob{ + inputFile: file, + addressExp: addressExp, + response: c, + }) + + if len(compileJobs) >= int(toCompileCount) { + go execBatchCompile(compileJobs) + } + jobMtx.Unlock() + + result := <-c + return result.code, result.address +} + +func compile(file, addressExp string) ([]byte, string) { + if useBatching { + return batchCompile(file, addressExp) + } + + fileExt := filepath.Ext(file) + outputFilePath := file[0:len(file)-len(fileExt)] + ".out" + compileFilePath := file[0:len(file)-len(fileExt)] + ".asmtemp" + + // Clean up files + defer os.Remove(outputFilePath) + defer os.Remove(compileFilePath) + + // First we are gonna load all the data from file and write it into temp file + // Technically this shouldn't be necessary but for some reason if the last line + // or the asm file has one of more spaces at the end and no new line, the last + // instruction is ignored and not compiled + buildTempAsmFile(file, addressExp, compileFilePath, "") + + fileDir := filepath.Dir(file) + + const asCmdLinux string = "powerpc-eabi-as" + const objcopyCmdLinux string = "powerpc-eabi-objcopy" + + // Set base args + args := []string{"-a32", "-mbig", "-mregnames", "-mgekko"} + + // If defsym is defined, add it to the args + if argConfig.DefSym != "" { + args = append(args, "-defsym", argConfig.DefSym) + } + + // Add paths to look at when resolving includes + args = append(args, "-I", fileDir, "-I", argConfig.ProjectRoot) + + // Set output file + args = append(args, "-o", outputFilePath, compileFilePath) + + cmd := exec.Command(asCmdLinux, args...) + + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Failed to compile file: %s\n", file) + fmt.Printf("%s", output) + panic("as failure") + } + + cmd = exec.Command(objcopyCmdLinux, "-O", "binary", outputFilePath, outputFilePath) + output, err = cmd.CombinedOutput() + if err != nil { + fmt.Printf("Failed to pull out .text section: %s\n", file) + fmt.Printf("%s", output) + panic("objcopy failure") + } + contents, err := ioutil.ReadFile(outputFilePath) + if err != nil { + log.Panicf("Failed to read compiled file %s\n%s\n", file, err.Error()) + } + + code := contents[:len(contents)-4] + address := contents[len(contents)-4:] + if address[0] != 0x80 && address[0] != 0x81 { + log.Panicf( + "Injection address in file %s evaluated to a value that does not start with 0x80 or 0x81"+ + ", probably an invalid address\n", + file, + ) + } + + return code, fmt.Sprintf("%x", address) +} + +func isSymbolRune(r rune) bool { + return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' +} + +func splitAny(s string, seps string) []string { + splitter := func(r rune) bool { + return strings.ContainsRune(seps, r) + } + return strings.FieldsFunc(s, splitter) +} + +func removeComments(asmContents []byte) []byte { + body := string(asmContents) + + // Remove block comments + newBody := "" + idx := strings.Index(body, "/*") + for idx > -1 { + newBody += body[:idx] + body = body[idx:] + end := strings.Index(body, "*/") + if end > -1 { + body = body[end+2:] + } + idx = strings.Index(body, "/*") + } + newBody += body + + // Remove line comments + lines := strings.Split(newBody, "\n") + newLines := []string{} + for _, line := range lines { + // Remove any comments + commentSplit := strings.Split(line, "#") + if len(commentSplit) == 0 { + newLines = append(newLines, line) + continue + } + newLines = append(newLines, strings.TrimSpace(commentSplit[0])) + } + + return []byte(strings.Join(newLines, "\n")) +} + +func isolateLabelNames(asmContents []byte) []byte { + // Start logic to isolate label names + // First we're going to extract all label positions as well as replace them with a number + // based label which functions as a local label. This will prevent errors from using + // the same label name in multiple files + lines := strings.Split(string(asmContents), "\n") + labels := map[string]labelInfo{} + newLines := []string{} + labelIdx := 100 // Start at 100 because hopefully no macros will use labels that high + for lineNum, line := range lines { + isLabel := len(line) > 0 && line[len(line)-1] == ':' + if !isLabel { + newLines = append(newLines, line) + continue + } + + name := line[:len(line)-1] + labels[name] = labelInfo{name, labelIdx, lineNum} + newLines = append(newLines, fmt.Sprintf("%d:", labelIdx)) + labelIdx += 1 + } + + // Now let's convert all the branch instructions we can find to use the local labels + // instead of the original label names + // TODO: It might be possible to throw errors here if referencing a label that doesn't exist + // TODO: I didn't do it yet because currently instructions like `branchl r12, ...` might + // TODO: trigger the easy form of detection. We'd probably have to detect all possible branch + // TODO: instructions in order to do this + finalLines := []string{} + for lineNum, line := range newLines { + parts := splitAny(line, " \t") + if len(parts) == 0 { + finalLines = append(finalLines, line) + continue + } + + label := parts[len(parts)-1] + li, labelExists := labels[label] + isBranch := len(parts) >= 2 && line[0] == 'b' && labelExists + if !isBranch { + finalLines = append(finalLines, line) + continue + } + + dir := "f" + if lineNum > li.linePos { + dir = "b" + } + + parts[len(parts)-1] = fmt.Sprintf("%d%s", li.num, dir) + finalLines = append(finalLines, strings.Join(parts, " ")) + } + + return []byte(strings.Join(finalLines, "\n")) +} + +func isolateSymbolNames(asmContents []byte, section string) []byte { + lines := strings.Split(string(asmContents), "\n") + symbolMap := map[string][]symbolInfo{} + newLines := []string{} + for idx, line := range lines { + parts := splitAny(line, " \t,") + if len(parts) == 0 { + newLines = append(newLines, line) + continue + } + + isSet := parts[0] == ".set" && len(parts) >= 3 + if !isSet { + newLines = append(newLines, line) + continue + } + + newSymbol := fmt.Sprintf("__%s_symbol_%d", section, idx) + + // Add this symbol to map + _, exists := symbolMap[parts[1]] + if !exists { + symbolMap[parts[1]] = []symbolInfo{} + } + symbolMap[parts[1]] = append(symbolMap[parts[1]], symbolInfo{newSymbol, idx}) + + newLines = append(newLines, strings.Replace(line, parts[1], newSymbol, 1)) + } + + finalLines := []string{} + for lineIdx, line := range newLines { + if len(line) == 0 { + finalLines = append(finalLines, line) + continue + } + + symbolParts := strings.FieldsFunc(line, func(r rune) bool { return !isSymbolRune(r) }) + connectingParts := strings.FieldsFunc(line, isSymbolRune) + + for symbolIdx, symbol := range symbolParts { + instances, exists := symbolMap[symbol] + if !exists { + continue + } + + remap := instances[0].name + for _, instance := range instances { + if instance.linePos > lineIdx { + break + } + remap = instance.name + } + + symbolParts[symbolIdx] = remap + } + + reconnected := []string{} + first, second := symbolParts, connectingParts + shouldSwap := !isSymbolRune(rune(line[0])) + if shouldSwap { + first, second = connectingParts, symbolParts + } + + for partIdx, part1 := range first { + reconnected = append(reconnected, part1) + if partIdx < len(second) { + reconnected = append(reconnected, second[partIdx]) + } + } + + finalLines = append(finalLines, strings.Join(reconnected, "")) + } + + return []byte(strings.Join(finalLines, "\n")) +} + +func buildTempAsmFile(sourceFilePath, addressExp, targetFilePath, section string) { + asmContents, err := ioutil.ReadFile(sourceFilePath) + if err != nil { + log.Panicf("Failed to read asm file: %s\n%s\n", sourceFilePath, err.Error()) + } + + // If section provided, we need to take some precautions to isolate the code from others + if section != "" { + // Add the section label at the top so the code can be extracted individually + asmContents = append([]byte(fmt.Sprintf(".section %s\n", section)), asmContents...) + + asmContents = removeComments(asmContents) + asmContents = isolateLabelNames(asmContents) + asmContents = isolateSymbolNames(asmContents, section) + } + + // Add new line before .set for address + asmContents = append(asmContents, []byte("\n")...) + + // Add .set to get file injection address + setLine := fmt.Sprintf(".long %s\n", addressExp) + asmContents = append(asmContents, []byte(setLine)...) + + // Explicitly add a new line at the end of the file, which should prevent line skip + asmContents = append(asmContents, []byte("\n")...) + err = ioutil.WriteFile(targetFilePath, asmContents, 0644) + if err != nil { + log.Panicf("Failed to write temporary asm file: %s\n%s\n", targetFilePath, err.Error()) + } +} diff --git a/gecko.go b/gecko.go index e4c49ab..b5ad034 100644 --- a/gecko.go +++ b/gecko.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "bytes" "encoding/hex" "encoding/json" "flag" @@ -14,6 +13,7 @@ import ( "path/filepath" "sort" "strings" + "sync/atomic" "time" ) @@ -76,7 +76,7 @@ var output []string func timeTrack(start time.Time) { elapsed := time.Since(start) - fmt.Printf("Process time was %s\n", elapsed) + fmt.Printf("Compiled %d files. Process time was %s\n", toCompileCount, elapsed) } func main() { @@ -98,6 +98,30 @@ func main() { // Ensure assembler files can be found confirmAssembler() + addDefsymFlag := func(fs *flag.FlagSet) *string { + return fs.String( + "defsym", + "", + "Allows the defining of symbols from the command line. Example: \"EX_SYM1=10,EX_SYM2=0xABC\"", + ) + } + + addIsRecursiveFlag := func(fs *flag.FlagSet) *bool { + return fs.Bool( + "r", + true, + "If true, will recursively find all .asm files within the sub-directories as well as the root directory.", + ) + } + + addBatchedFlag := func(fs *flag.FlagSet) *bool { + return fs.Bool( + "batched", + false, + "If true, all files will be batched and assembled together. This does have some quirks, visit github for details.", + ) + } + command := os.Args[1] switch command { case "build": @@ -112,13 +136,12 @@ func main() { "", "Additional output file path. Using a .gct extension will output a gct. Everything else will output text. Will be appended to the files in the config file.", ) - defsymPtr := buildFlags.String( - "defsym", - "", - "Allows the defining of symbols from the command line. Example: \"EX_SYM1=10,EX_SYM2=0xABC\"", - ) + defsymPtr := addDefsymFlag(buildFlags) + batchedPtr := addBatchedFlag(buildFlags) buildFlags.Parse(os.Args[2:]) + useBatching = *batchedPtr + config := readConfigFile(*configFilePathPtr) outputFiles = config.OutputFiles if *outputFilePtr != "" { @@ -137,6 +160,7 @@ func main() { argConfig.ProjectRoot = projectRootTemp argConfig.DefSym = *defsymPtr + countFilesToCompile(config) buildBody(config) case "assemble": assembleFlags := flag.NewFlagSet("assemble", flag.ExitOnError) @@ -150,18 +174,13 @@ func main() { ".", "The root directory to assemble. Will default to the current directory.", ) - isRecursivePtr := assembleFlags.Bool( - "r", - true, - "If true, will recursively find all .asm files within the sub-directories as well as the root directory.", - ) - defsymPtr := assembleFlags.String( - "defsym", - "", - "Allows the defining of symbols from the command line. Example: \"EX_SYM1=10,EX_SYM2=0xABC\"", - ) + isRecursivePtr := addIsRecursiveFlag(assembleFlags) + defsymPtr := addDefsymFlag(assembleFlags) + batchedPtr := addBatchedFlag(assembleFlags) assembleFlags.Parse(os.Args[2:]) + useBatching = *batchedPtr + configDir := filepath.Dir(*assemblePathPtr) projectRootTemp, err := filepath.Abs(configDir) if err != nil { @@ -171,6 +190,10 @@ func main() { argConfig.ProjectRoot = projectRootTemp argConfig.DefSym = *defsymPtr + // Calculate the number of files that will be compiled + asmFilePaths := collectFilesFromFolder(*assemblePathPtr, *isRecursivePtr) + atomic.AddUint32(&toCompileCount, uint32(len(asmFilePaths))) + outputFiles = append(outputFiles, FileDetails{File: *outputFilePtr}) output = generateInjectionFolderLines(*assemblePathPtr, *isRecursivePtr) case "list": @@ -185,11 +208,7 @@ func main() { "injection-list.json", "Output file name where the list will be saved.", ) - isRecursivePtr := listFlags.Bool( - "r", - true, - "If true, will recursively find all .asm files within the sub-directories as well as the root directory.", - ) + isRecursivePtr := addIsRecursiveFlag(listFlags) listFlags.Parse(os.Args[2:]) listInjections(*inputPtr, *outputFilePtr, *isRecursivePtr) case "-h": @@ -211,6 +230,8 @@ func main() { for _, file := range outputFiles { writeOutput(file) } + + compileWaitGroup.Wait() } func readConfigFile(path string) Config { @@ -231,6 +252,20 @@ func readConfigFile(path string) Config { return result } +func countFilesToCompile(config Config) { + for _, desc := range config.Codes { + for _, geckoCode := range desc.Build { + switch geckoCode.Type { + case Inject: + atomic.AddUint32(&toCompileCount, 1) + case InjectFolder: + asmFilePaths := collectFilesFromFolder(geckoCode.SourceFolder, geckoCode.IsRecursive) + atomic.AddUint32(&toCompileCount, uint32(len(asmFilePaths))) + } + } + } +} + func buildBody(config Config) { // go through every code and print a header and the codes that make it up resultsChan := make(chan lineAggregateResult, len(config.Codes)) @@ -697,96 +732,6 @@ func confirmAssembler() { } } -func compile(file, addressExp string) ([]byte, string) { - fileExt := filepath.Ext(file) - outputFilePath := file[0:len(file)-len(fileExt)] + ".out" - compileFilePath := file[0:len(file)-len(fileExt)] + ".asmtemp" - - // Clean up files - defer os.Remove(outputFilePath) - defer os.Remove(compileFilePath) - - // First we are gonna load all the data from file and write it into temp file - // Technically this shouldn't be necessary but for some reason if the last line - // or the asm file has one of more spaces at the end and no new line, the last - // instruction is ignored and not compiled - buildTempAsmFile(file, addressExp, compileFilePath) - - fileDir := filepath.Dir(file) - - const asCmdLinux string = "powerpc-eabi-as" - const objcopyCmdLinux string = "powerpc-eabi-objcopy" - - // Set base args - args := []string{"-a32", "-mbig", "-mregnames"} - - // If defsym is defined, add it to the args - if argConfig.DefSym != "" { - args = append(args, "-defsym", argConfig.DefSym) - } - - // Add paths to look at when resolving includes - args = append(args, "-I", fileDir, "-I", argConfig.ProjectRoot) - - // Set output file - args = append(args, "-o", outputFilePath, compileFilePath) - - cmd := exec.Command(asCmdLinux, args...) - - output, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("Failed to compile file: %s\n", file) - fmt.Printf("%s", output) - panic("as failure") - } - - contents, err := ioutil.ReadFile(outputFilePath) - if err != nil { - log.Panicf("Failed to read compiled file %s\n%s\n", file, err.Error()) - } - - // This gets the index right before the value of the last .set - addressEndIndex := bytes.LastIndex(contents, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xF1, 0x00}) - address := contents[addressEndIndex-4 : addressEndIndex] - if address[0] != 0x80 { - log.Panicf("Injection address in file %s evaluated to a value that does not start with 0x80, probably an invalid address\n", file) - } - - cmd = exec.Command(objcopyCmdLinux, "-O", "binary", outputFilePath, outputFilePath) - output, err = cmd.CombinedOutput() - if err != nil { - fmt.Printf("Failed to pull out .text section: %s\n", file) - fmt.Printf("%s", output) - panic("objcopy failure") - } - contents, err = ioutil.ReadFile(outputFilePath) - if err != nil { - log.Panicf("Failed to read compiled file %s\n%s\n", file, err.Error()) - } - return contents, fmt.Sprintf("%x", address) -} - -func buildTempAsmFile(sourceFilePath, addressExp, targetFilePath string) { - asmContents, err := ioutil.ReadFile(sourceFilePath) - if err != nil { - log.Panicf("Failed to read asm file: %s\n%s\n", sourceFilePath, err.Error()) - } - - // Add new line before .set for address - asmContents = append(asmContents, []byte("\r\n")...) - - // Add .set to get file injection address - setLine := fmt.Sprintf(".set GTI_FILE_INJECTION_ADDRESS, %s", addressExp) - asmContents = append(asmContents, []byte(setLine)...) - - // Explicitly add a new line at the end of the file, which should prevent line skip - asmContents = append(asmContents, []byte("\r\n")...) - err = ioutil.WriteFile(targetFilePath, asmContents, 0644) - if err != nil { - log.Panicf("Failed to write temporary asm file\n%s\n", err.Error()) - } -} - func writeOutput(details FileDetails) { fmt.Printf("Writing to %s...\n", details.File) ext := filepath.Ext(details.File)