diff --git a/cmd/build.go b/cmd/build.go index 9f03199..8da9505 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -77,12 +77,12 @@ func buildCommand(cmd *cobra.Command, args []string) error { */ extension := strings.ToLower(strings.TrimLeft(filepath.Ext(recipePath), ".")) if len(extension) == 0 || (extension != "yml" && extension != "yaml") { - return fmt.Errorf("%s is an invalid recipe file", recipePath) + return fmt.Errorf("Recipe `%s` is an invalid recipe file", recipePath) } // Check whether the provided file exists, if not, then return an error if _, err := os.Stat(recipePath); errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("%s does not exist", recipePath) + return fmt.Errorf("Recipe `%s` does not exist", recipePath) } } diff --git a/core/build.go b/core/build.go index d5879a7..910a396 100644 --- a/core/build.go +++ b/core/build.go @@ -3,6 +3,7 @@ package core import ( "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -11,6 +12,11 @@ import ( "github.com/vanilla-os/vib/api" ) +var modulesCount int +var includeDepth int +var maxIncludeDepth = 1 +var errorCount = 0 + // Add a WORKDIR instruction to the containerfile func ChangeWorkingDirectory(workdir string, containerfile *os.File) error { if workdir != "" { @@ -46,7 +52,7 @@ func BuildRecipe(recipePath string, arch string, containerfilePath string) (api. return api.Recipe{}, err } - fmt.Printf("Building recipe %s\n", recipe.Name) + fmt.Printf("Building recipe `%s`\n", recipe.Name) // assuming the Containerfile location is relative if len(containerfilePath) == 0 { @@ -92,7 +98,7 @@ func BuildContainerfile(recipe *api.Recipe, arch string) error { // build the modules* // * actually just build the commands that will be used // in the Containerfile to build the modules - cmds, err := BuildModules(recipe, stage.Modules, stage.Cleanup, arch) + cmds, err := BuildModules(recipe, stage.Modules, stage.Cleanup, arch, stage.Id) if err != nil { return err } @@ -195,7 +201,7 @@ func BuildContainerfile(recipe *api.Recipe, arch string) error { cleanupSuffix := api.GetCleanupSuffix(stage.Cleanup) for _, cmd := range stage.Runs.Commands { _, err = containerfile.WriteString( - fmt.Sprintf("RUN %s\n", cmd + cleanupSuffix), + fmt.Sprintf("RUN %s\n", cmd+cleanupSuffix), ) if err != nil { return err @@ -313,44 +319,177 @@ func BuildContainerfile(recipe *api.Recipe, arch string) error { return nil } +func ExhaustCollectedErrors(_errors *[]error) int { + length := len(*_errors) + if length == 0 { + return 0 + } + + for _, err := range *_errors { + fmt.Printf("%v\n", err) + } + + *_errors = nil + errorCount += length + return length +} + +func MapSlicesToInterfaceSlices(inter []map[string]interface{}) []interface{} { + result := make([]interface{}, len(inter)) + for i, m := range inter { + result[i] = m + } + return result +} + +func MapToInterfaceSlices(m map[string]interface{}) []interface{} { + panic("If you need to use this function, you're likely handling module interfaces wrong") + // result := make([]interface{}, 0, len(m)) + // for _, v := range m { + // result = append(result, v) + // } + // return result +} + +func DecodeModuleToGenericModule(module interface{}, errors *[]error) (Module, error) { + var decodedModule Module + var customErr error = nil + defaultErr := mapstructure.Decode(module, &decodedModule) + + if defaultErr != nil { + customErr = fmt.Errorf("error: yaml decode error: failed to decode module to generic module with further error: %v", defaultErr) + (*errors) = append((*errors), customErr) + } + + return decodedModule, customErr +} + +func CollectModulesRecursively(modules []interface{}, allModules *[]interface{}, occurances *map[string][]int, errors *[]error) { + for _, module := range modules { + modulesCount++ + + decodedModule, err := DecodeModuleToGenericModule(module, errors) + if err != nil { + continue + } + + c := 0 + if decodedModule.Name == "" { + c |= 0b10 + } + if decodedModule.Type == "" { + c |= 0b01 + } + + switch c { + case 0b11: + fmt.Printf("error: module name and type cannot be == \"\"") + continue + case 0b10: + fmt.Printf("error: module name cannot be == \"\"") + continue + case 0b01: + fmt.Printf("error: module type cannot be == \"\"") + continue + case 0b00: + // fallthrough + } + + *allModules = append(*allModules, module) + (*occurances)[decodedModule.Name] = append((*occurances)[decodedModule.Name], modulesCount) + + ExhaustCollectedErrors(errors) + + if len(decodedModule.Modules) > 0 { + CollectModulesRecursively(MapSlicesToInterfaceSlices(decodedModule.Modules), allModules, occurances, errors) + } + } +} + // Build commands for each module in the recipe -func BuildModules(recipe *api.Recipe, modules []interface{}, cleanup []string, arch string) ([]ModuleCommand, error) { +func BuildModules(recipe *api.Recipe, modules []interface{}, cleanup []string, arch string, stageName string) ([]ModuleCommand, error) { + var _errors []error + var allModules []interface{} + modNameOccursInMod := make(map[string][]int) + + CollectModulesRecursively(modules, &allModules, &modNameOccursInMod, &_errors) + cmds := []ModuleCommand{} + for _, moduleInterface := range modules { - var module Module - err := mapstructure.Decode(moduleInterface, &module) + decodedModule, cmd, err := BuildModule(recipe, moduleInterface, &allModules, &modNameOccursInMod, cleanup, arch, stageName, &_errors) if err != nil { - return nil, err - } + if !(decodedModule.Type == "includes" && (errors.Is(err, os.ErrNotExist) || errors.Is(err, fs.ErrNotExist))) { + _errors = append(_errors, err) + } + ExhaustCollectedErrors(&_errors) + fmt.Printf("Building [%s] module `%s`: failed\n", decodedModule.Type, decodedModule.Name) - cmd, err := BuildModule(recipe, moduleInterface, cleanup, arch) - if err != nil { - return nil, err + continue } + ExhaustCollectedErrors(&_errors) + cmds = append(cmds, ModuleCommand{ - Name: module.Name, + Name: decodedModule.Name, Command: append(cmd, ""), // add empty entry to ensure proper newline in Containerfile - Workdir: module.Workdir, + Workdir: decodedModule.Workdir, }) } + for _, occurancesInMods := range modNameOccursInMod { + occurances := len(occurancesInMods) + + if occurances > 1 { + decodedModule, err := DecodeModuleToGenericModule(allModules[occurancesInMods[0]], &_errors) + if err != nil { + panic("This module was previously decoded but now fails to. Needs fix in vib codebase.") + } + _errors = append(_errors, fmt.Errorf("error: found ambiguous module with name `%s` %d times:", decodedModule.Name, occurances)) + + for j := range occurances { + if j > 0 { + decodedModule, err = DecodeModuleToGenericModule(allModules[occurancesInMods[j]], &_errors) + } else if err != nil { + continue + } + _errors = append(_errors, fmt.Errorf("note: found in file `%s`", decodedModule.Workdir)) + // TODO: This is not the correct variable to get the file path of the module, which we should display. + } + ExhaustCollectedErrors(&_errors) + errorCount += occurances + } + } + + if errorCount > 0 { + return nil, fmt.Errorf("Encoutered %d errors while building %d modules\n", errorCount, modulesCount) + } + return cmds, nil } -func buildIncludesModule(moduleInterface interface{}, recipe *api.Recipe, cleanup []string, arch string) (string, error) { - var include IncludesModule - err := mapstructure.Decode(moduleInterface, &include) - if err != nil { - return "", err +func BuildIncludesModule(recipe *api.Recipe, module interface{}, allModules *[]interface{}, occurances *map[string][]int, cleanup []string, arch string, stageName string, _errors *[]error) (string, error) { + // Note: errors is called _errors here because this function needs the errors package. + + includeDepth++ + defer func() { includeDepth-- }() + + var includeModule IncludesModule + if _err := mapstructure.Decode(module, &includeModule); _err != nil { + return "", _err } - if len(include.Includes) == 0 { - return "", errors.New("includes module must have at least one module to include") + if includeDepth > 1 { + return "", fmt.Errorf("[includes] module nesting is currently limited to `%d` layers.\n Found includes module in `%s`\n", maxIncludeDepth, includeModule.Name) + } + + if len(includeModule.Includes) == 0 { + return "", fmt.Errorf("[includes] module `%s` must have at least one module to include", includeModule.Name) } var commands []string - for _, include := range include.Includes { + var err error = nil + for _, include := range includeModule.Includes { var modulePath string // in case of a remote include, we need to download the @@ -359,6 +498,7 @@ func buildIncludesModule(moduleInterface interface{}, recipe *api.Recipe, cleanu fmt.Printf("Downloading recipe from %s\n", include) modulePath, err = downloadRecipe(include) if err != nil { + *_errors = append(*_errors, err) return "", err } } else if followsGhPattern(include) { @@ -367,75 +507,129 @@ func buildIncludesModule(moduleInterface interface{}, recipe *api.Recipe, cleanu fmt.Printf("Downloading recipe from %s\n", include) modulePath, err = downloadGhRecipe(include) if err != nil { - return "", err + *_errors = append(*_errors, err) + continue } } else { modulePath = filepath.Join(recipe.ParentPath, include) } - includeModule, err := GenModule(modulePath) - if err != nil { - return "", err - } + generatedModule, _err := GenModule(modulePath) - buildModule, err := BuildModule(recipe, includeModule, cleanup, arch) - if err != nil { - return "", err + if errors.Is(_err, os.ErrNotExist) || errors.Is(_err, fs.ErrNotExist) { + customErr := fmt.Errorf("error: [%s] module `%s` includes\n `%s`,\n which doesn't exist", includeModule.Type, includeModule.Name, modulePath) + + (*_errors) = append(*_errors, customErr) + (*_errors) = append(*_errors, _err) + + err = _err + continue + } else if _err != nil { + (*_errors) = append(*_errors, _err) + return "", _err } - commands = append(commands, buildModule...) - } - return strings.Join(commands, "\n"), nil -} -// Build a command string for the given module in the recipe -func BuildModule(recipe *api.Recipe, moduleInterface interface{}, cleanup []string, arch string) ([]string, error) { - var module Module - err := mapstructure.Decode(moduleInterface, &module) - if err != nil { - return []string{""}, err - } + ExhaustCollectedErrors(_errors) - fmt.Printf("Building module [%s] of type [%s]\n", module.Name, module.Type) + var _errors []error // temporary - commands := []string{fmt.Sprintf("\n# Begin Module %s - %s", module.Name, module.Type)} + decodedModule, cmd, _err := BuildModule(recipe, generatedModule, allModules, occurances, cleanup, arch, stageName, &_errors) + if _err != nil { + // _errors = append(_errors, _err) + ExhaustCollectedErrors(&_errors) + err = _err + continue + } - if len(module.Modules) > 0 { - for _, nestedModule := range module.Modules { - buildModule, err := BuildModule(recipe, nestedModule, append(cleanup, module.Cleanup...), arch) - if err != nil { - return []string{""}, err + commands = append(commands, cmd...) + *allModules = append(*allModules, decodedModule) + (*occurances)[decodedModule.Name] = append((*occurances)[decodedModule.Name], modulesCount) + + fmt.Printf("Building all %d submodules of [%s] module `%s` included in `%s`\n", len(decodedModule.Modules), decodedModule.Type, decodedModule.Name, includeModule.Name) + var failed bool = false + includeModuleIdx := len(*allModules) + CollectModulesRecursively(MapSlicesToInterfaceSlices(decodedModule.Modules), allModules, occurances, &_errors) + + modulesLeftToBuild := len(*allModules) - includeModuleIdx + + for i := modulesLeftToBuild; i > 0; i-- { + _, buildModule, _err := BuildModule(recipe, (*allModules)[len(*allModules)-i], allModules, occurances, cleanup, arch, stageName, &_errors) + if _err != nil { + ExhaustCollectedErrors(&_errors) + fmt.Printf("%d/%d Building [%s] module of submodule `%s` included in `%s`: failed\n", i, len(decodedModule.Modules), decodedModule.Type, decodedModule.Name, includeModule.Name) + failed = true + err = _err + continue } + commands = append(commands, buildModule...) } + + ExhaustCollectedErrors(&_errors) + if failed { + fmt.Printf("Building all %d submodules of [%s] module `%s` included in `%s`: failed\n", len(decodedModule.Modules), decodedModule.Type, decodedModule.Name, includeModule.Name) + } else { + fmt.Printf("Buildung all %d submodules of [%s] module `%s` included in `%s`: success\n", len(decodedModule.Modules), decodedModule.Type, decodedModule.Name, includeModule.Name) + } } + return strings.Join(commands, "\n"), err +} - moduleBuilders := map[string]func(interface{}, *api.Recipe, []string, string) (string, error){ - "shell": BuildShellModule, - "includes": buildIncludesModule, +// Build a command string for the given module in the recipe +func BuildModule(recipe *api.Recipe, module interface{}, allModules *[]interface{}, occurances *map[string][]int, cleanup []string, arch string, stageName string, _errors *[]error) (Module, []string, error) { + decodedModule, err := DecodeModuleToGenericModule(module, _errors) + if err != nil { + return decodedModule, []string{""}, err } - if moduleBuilder, ok := moduleBuilders[module.Type]; ok { - command, err := moduleBuilder(moduleInterface, recipe, cleanup, arch) + commands := []string{fmt.Sprintf("\n# Begin Module %s - %s", decodedModule.Name, decodedModule.Type)} + defer func() { + commands = append(commands, fmt.Sprintf("# End Module %s - %s\n", decodedModule.Name, decodedModule.Type)) + }() + + fmt.Printf("Building [%s] module `%s`\n", decodedModule.Type, decodedModule.Name) + + switch decodedModule.Type { + case "shell": + command, err := BuildShellModule(module, recipe, cleanup, arch) if err != nil { - return []string{""}, err + return decodedModule, []string{""}, err } commands = append(commands, command) - } else { - command, err := LoadBuildPlugin(module.Type, moduleInterface, recipe, cleanup, arch) + case "includes": + command, err := BuildIncludesModule(recipe, module, allModules, occurances, cleanup, arch, stageName, _errors) + if err != nil { + return decodedModule, []string{""}, err + } + commands = append(commands, command) + case "": + err := fmt.Errorf("error: module `%s` tried to use a plugin but specified no name", decodedModule.Name) + return decodedModule, []string{""}, err + default: + command, err := LoadBuildPlugin(decodedModule.Type, module, recipe, cleanup, arch) if err != nil { - return []string{""}, err + return decodedModule, []string{""}, err } commands = append(commands, command...) } - moduleSourcePath := filepath.Join(recipe.SourcesPath, module.Name) - err = os.MkdirAll(moduleSourcePath, 0755) + moduleSourcePath := filepath.Join(recipe.SourcesPath, decodedModule.Name) + _ = os.MkdirAll(moduleSourcePath, 0755) + sourcePath := filepath.Join(recipe.SourcesPath, decodedModule.Name) + stageSourcePath := filepath.Join(recipe.SourcesPath, stageName, decodedModule.Name) + + _ = os.MkdirAll(sourcePath, 0o777) + _ = os.MkdirAll(filepath.Dir(stageSourcePath), 0o777) + + err = os.Rename(sourcePath, stageSourcePath) if err != nil { - return []string{""}, err + if errors.Is(err, os.ErrExist) || errors.Is(err, fs.ErrExist) { + fmt.Printf("Multiple module name error!\n") + return decodedModule, []string{}, nil + } + return decodedModule, []string{}, fmt.Errorf("could not rename `%s` to `%s`: %w\n", sourcePath, stageSourcePath, err) } - commands = append(commands, fmt.Sprintf("# End Module %s - %s\n", module.Name, module.Type)) - - fmt.Printf("Module [%s] built successfully\n", module.Name) - return commands, nil + fmt.Printf("Building [%s] module `%s`: success\n", decodedModule.Type, decodedModule.Name) + return decodedModule, commands, nil } diff --git a/core/loader.go b/core/loader.go index 7a66cab..3c21941 100644 --- a/core/loader.go +++ b/core/loader.go @@ -156,6 +156,10 @@ func downloadRecipe(url string) (path string, err error) { } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return "", fmt.Errorf("error: resource not found: %s", url) + } + tmpFile, err := os.CreateTemp("", "vib-recipe-") if err != nil { return "", err diff --git a/core/plugins.in b/core/plugins.in index f9ce1c5..496c100 100644 --- a/core/plugins.in +++ b/core/plugins.in @@ -11,8 +11,8 @@ import ( "github.com/vanilla-os/vib/api" ) import ( - "errors" "encoding/base64" + "errors" "os" "syscall" ) @@ -34,10 +34,12 @@ func decodeBuildCmds(cmds string) ([]string, error) { } func LoadPlugin(name string, plugintype api.PluginType, recipe *api.Recipe) (uintptr, api.PluginInfo, error) { - fmt.Println("Loading new plugin") + if len(name) == 0 { + panic("Cannot load a module without its name. Needs a fix in the codebase.") + } + fmt.Printf("Loading plugin [%s]", name) projectPluginPath := fmt.Sprintf("%s/%s.so", recipe.PluginPath, name) - installPrefixPath := fmt.Sprintf("%INSTALLPREFIX%/share/vib/plugins/%s.so", name) globalPluginPathsEnv, isXDDDefined := os.LookupEnv("XDG_DATA_DIRS") @@ -65,24 +67,31 @@ func LoadPlugin(name string, plugintype api.PluginType, recipe *api.Recipe) (uin // of paths to search. var _errors = make([]error, len(allPluginPaths)) + var _err error = nil + for index, path := range allPluginPaths { _, err := os.Stat(path) if err != nil { _errors = append(_errors, err) + if index == lastIndex { - // If the last available path doesn't exist, - // panic with all the error messages. - panic(errors.Join(_errors...)) - } + _errors = append(_errors, fmt.Errorf("error: couldn't find plugin [%s] on your system.\nnote: Please copy it into one of the searched folders above.", name)) - continue + for _, _err := range _errors { + fmt.Printf("%v\n", _err) + } + _err = err + break + } else { + continue + } } loadedPlugin, err = purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) if err != nil { _errors = append(_errors, err) if index == lastIndex { - // If the last available plugin can't be loaded, + // If the last available plugin path can't be loaded, // panic with all the error messages. panic(errors.Join(_errors...)) } @@ -123,7 +132,7 @@ func LoadPlugin(name string, plugintype api.PluginType, recipe *api.Recipe) (uin } } - return loadedPlugin, *pluginInfo, nil + return loadedPlugin, *pluginInfo, _err } func LoadBuildPlugin(name string, moduleInterface interface{}, recipe *api.Recipe, cleanup []string, arch string) ([]string, error) { @@ -152,8 +161,8 @@ func LoadBuildPlugin(name string, moduleInterface interface{}, recipe *api.Recip buildModule.PluginInfo = pluginInfo openedBuildPlugins[name] = buildModule } - fmt.Printf("Using plugin: %s\n", buildModule.Name) - moduleJson, err := json.Marshal(moduleInterface) + fmt.Printf("Using plugin [%s]\n", buildModule.Name) + moduleJson, err := json.Marshal(module) if err != nil { return []string{""}, err } diff --git a/core/shell.go b/core/shell.go index e09329a..2483342 100644 --- a/core/shell.go +++ b/core/shell.go @@ -1,7 +1,6 @@ package core import ( - "errors" "fmt" "strings" @@ -21,21 +20,21 @@ type ShellModule struct { // Build shell module commands and return them as a single string // // Returns: Concatenated shell commands or an error if any step fails -func BuildShellModule(moduleInterface interface{}, recipe *api.Recipe, cleanup []string, arch string) (string, error) { - var module ShellModule - err := mapstructure.Decode(moduleInterface, &module) - if err != nil { +func BuildShellModule(module interface{}, recipe *api.Recipe, cleanup []string, arch string) (string, error) { + var shellModule ShellModule + + if err := mapstructure.Decode(module, &shellModule); err != nil { return "", err } - for _, source := range module.Sources { + for _, source := range shellModule.Sources { if api.TestArch(source.OnlyArches, arch) { if strings.TrimSpace(source.Type) != "" { - err := api.DownloadSource(recipe, source, module.Name) + err := api.DownloadSource(recipe, source, shellModule.Name) if err != nil { return "", err } - err = api.MoveSource(recipe.DownloadsPath, recipe.SourcesPath, source, module.Name) + err = api.MoveSource(recipe.DownloadsPath, recipe.SourcesPath, source, shellModule.Name) if err != nil { return "", err } @@ -43,18 +42,22 @@ func BuildShellModule(moduleInterface interface{}, recipe *api.Recipe, cleanup [ } } - if len(module.Commands) == 0 { - return "", errors.New("no commands specified") + if len(shellModule.Commands) == 0 { + return "", fmt.Errorf("no commands specified") } - cmd := "" - for i, command := range module.Commands { - cmd += command - if i < len(module.Commands)-1 { - cmd += " && " + var cmd strings.Builder + _, err := fmt.Fprintf(&cmd, "RUN --mount=source=sources/%s,target=/sources/%s,rw\nRUN ", shellModule.Name, shellModule.Name) + if err != nil { + panic(fmt.Sprintf("Fprintf failed during build of shell module `%s`", shellModule.Name)) + } + for i, command := range shellModule.Commands { + cmd.WriteString(command) + if i < len(shellModule.Commands)-1 { + cmd.WriteString(" && ") } } - cmd += api.GetCleanupSuffix(append(cleanup, module.Cleanup...)) + cmd.WriteString(api.GetCleanupSuffix(append(cleanup, shellModule.Cleanup...))) - return fmt.Sprintf("RUN --mount=source=sources/%s,target=/sources/%s,rw ", module.Name, module.Name) + cmd, nil + return cmd.String(), nil }