diff --git a/internal/imagedefinition/README.rst b/internal/imagedefinition/README.rst index 8a7d2d88..1f53693e 100644 --- a/internal/imagedefinition/README.rst +++ b/internal/imagedefinition/README.rst @@ -111,6 +111,13 @@ The following specification defines what is supported in the YAML: # An alternative branch to use while retrieving seeds # from a git or bzr source. branch: (optional) + # Whether to write the sources list as Deb822 formatted entries in + # /etc/apt/sources.list.d/ubuntu.sources or not (and thus use the legacy format + # in /etc/apt/sources.list) + # Default to "false" for now to not break existing builds but a warning will be + # displayed and this default will switch at some point in the future. + # A warning is also displayed if no value was explicitely set for this field. + sources-list-deb822: (optional) # Used for pre-built root filesystems rather than germinating # from a seed or using a list of archive-tasks. Must be an # an uncompressed tar archive or a tar archive with one of the diff --git a/internal/imagedefinition/image_definition.go b/internal/imagedefinition/image_definition.go index 7ea65659..8f708422 100644 --- a/internal/imagedefinition/image_definition.go +++ b/internal/imagedefinition/image_definition.go @@ -39,14 +39,15 @@ type Gadget struct { // Rootfs defines the rootfs section of the image definition file type Rootfs struct { - Components []string `yaml:"components" json:"Components,omitempty" default:"main,restricted"` - Archive string `yaml:"archive" json:"Archive" default:"ubuntu"` - Flavor string `yaml:"flavor" json:"Flavor" default:"ubuntu"` - Mirror string `yaml:"mirror" json:"Mirror" default:"http://archive.ubuntu.com/ubuntu/"` - Pocket string `yaml:"pocket" json:"Pocket" jsonschema:"enum=release,enum=Release,enum=updates,enum=Updates,enum=security,enum=Security,enum=proposed,enum=Proposed" default:"release"` - Seed *Seed `yaml:"seed" json:"Seed,omitempty" jsonschema:"oneof_required=Seed"` - Tarball *Tarball `yaml:"tarball" json:"Tarball,omitempty" jsonschema:"oneof_required=Tarball"` - ArchiveTasks []string `yaml:"archive-tasks" json:"ArchiveTasks,omitempty" jsonschema:"oneof_required=ArchiveTasks"` + Components []string `yaml:"components" json:"Components,omitempty" default:"main,restricted"` + Archive string `yaml:"archive" json:"Archive" default:"ubuntu"` + Flavor string `yaml:"flavor" json:"Flavor" default:"ubuntu"` + Mirror string `yaml:"mirror" json:"Mirror" default:"http://archive.ubuntu.com/ubuntu/"` + Pocket string `yaml:"pocket" json:"Pocket" jsonschema:"enum=release,enum=Release,enum=updates,enum=Updates,enum=security,enum=Security,enum=proposed,enum=Proposed" default:"release"` + Seed *Seed `yaml:"seed" json:"Seed,omitempty" jsonschema:"oneof_required=Seed"` + Tarball *Tarball `yaml:"tarball" json:"Tarball,omitempty" jsonschema:"oneof_required=Tarball"` + ArchiveTasks []string `yaml:"archive-tasks" json:"ArchiveTasks,omitempty" jsonschema:"oneof_required=ArchiveTasks"` + SourcesListDeb822 *bool `yaml:"sources-list-deb822" json:"SourcesListDeb822" default:"false"` } // Seed defines the seed section of rootfs, which is used to @@ -307,57 +308,184 @@ type DependentKeyError struct { gojsonschema.ResultErrorFields } -func (imageDef ImageDefinition) securityMirror() string { - if imageDef.Architecture == "amd64" || imageDef.Architecture == "i386" { +func (i ImageDefinition) securityMirror() string { + if i.Architecture == "amd64" || i.Architecture == "i386" { return "http://security.ubuntu.com/ubuntu/" } - return imageDef.Rootfs.Mirror + return i.Rootfs.Mirror } -func generatePocketList(series string, components []string, mirror string, securityMirror string, pocket string) []string { - baseList := fmt.Sprintf("deb %%s %s%%s %s\n", series, strings.Join(components, " ")) +// generateLegacySourcesList returns the content to write to the sources.list file +// under the legacy format. +func generateLegacySourcesList(series string, components []string, mirror string, securityMirror string, pocket string) string { + baseList := fmt.Sprintf("deb %%s %s%%s %s", series, strings.Join(components, " ")) - releaseList := fmt.Sprintf(baseList, mirror, "") - securityList := fmt.Sprintf(baseList, securityMirror, "-security") - updatesList := fmt.Sprintf(baseList, mirror, "-updates") - proposedList := fmt.Sprintf(baseList, mirror, "-proposed") + releaseSourceComment := `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +` + updatesSourceComment := `## Major bug fix updates produced after the final release of the +## distribution. +` - pocketList := make([]string, 0) + releaseSource := releaseSourceComment + fmt.Sprintf(baseList, mirror, "") + securitySource := fmt.Sprintf(baseList, securityMirror, "-security") + updatesSource := updatesSourceComment + fmt.Sprintf(baseList, mirror, "-updates") + proposedSource := fmt.Sprintf(baseList, mirror, "-proposed") + + sourcesList := make([]string, 0) switch pocket { case "release": - pocketList = append(pocketList, releaseList) + sourcesList = append(sourcesList, releaseSource) case "security": - pocketList = append(pocketList, releaseList, securityList) + sourcesList = append(sourcesList, releaseSource, securitySource) case "updates": - pocketList = append(pocketList, releaseList, securityList, updatesList) + sourcesList = append(sourcesList, releaseSource, securitySource, updatesSource) + case "proposed": + sourcesList = append(sourcesList, releaseSource, securitySource, updatesSource, proposedSource) + } + + return strings.Join(sourcesList, "\n") + "\n" +} + +// LegacyBuildSourcesList returns the content of the /etc/apt/sources.list to be used +// during the build process +func (i *ImageDefinition) LegacyBuildSourcesList() string { + return i.legacySourcesList(false) +} + +// LegacyTargetSourcesList returns the content of the /etc/apt/sources.list for the target +// image +func (i *ImageDefinition) LegacyTargetSourcesList() string { + return i.legacySourcesList(true) +} + +// legacySourcesList returns the content of the /etc/apt/sources.list file in the +// legacy format (not deb822). +func (i *ImageDefinition) legacySourcesList(target bool) string { + pocket := i.Rootfs.Pocket + if target { + pocket = i.Customization.Pocket + } + + return generateLegacySourcesList( + i.Series, + i.Customization.Components, + i.Rootfs.Mirror, + i.securityMirror(), + strings.ToLower(pocket)) +} + +// generateDeb822Section returns a deb822 section/paragraph to be used in a sources list file +// This function is tailored to what is expected in an official ubuntu image and should not be +// used as is to generate arbitrary deb822 sections. +func generateDeb822Section(mirror string, series string, components []string, pocket string) string { + sectionTmpl := `Types: deb +URIs: %s +Suites: %s +Components: %s +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +` + + suites := make([]string, 0) + + switch pocket { + case "security": + suites = []string{series + "-security"} case "proposed": - pocketList = append(pocketList, releaseList, securityList, updatesList, proposedList) + suites = append([]string{series + "-proposed"}, suites...) + fallthrough + case "updates": + suites = append([]string{series + "-updates"}, suites...) + fallthrough + case "release": + suites = append([]string{series}, suites...) } - return pocketList -} - -// BuildPocketList returns a slice of strings that need to be added to -// /etc/apt/sources.list in the chroot to build the image, based on the value of "pocket" -// in the rootfs section of the image definition -func (imageDef ImageDefinition) BuildPocketList() []string { - return generatePocketList( - imageDef.Series, - imageDef.Rootfs.Components, - imageDef.Rootfs.Mirror, - imageDef.securityMirror(), - strings.ToLower(imageDef.Rootfs.Pocket)) -} - -// TargetPocketList returns a slice of strings that need to be added to -// /etc/apt/sources.list in the chroot for the target image, based on the value of "pocket" -// in the customization section of the image definition -func (imageDef ImageDefinition) TargetPocketList() []string { - return generatePocketList( - imageDef.Series, - imageDef.Customization.Components, - imageDef.Rootfs.Mirror, - imageDef.securityMirror(), - strings.ToLower(imageDef.Customization.Pocket)) + return fmt.Sprintf(sectionTmpl, + mirror, + strings.Join(suites, " "), + strings.Join(components, " "), + ) +} + +var LegacySourcesListComment = `# Ubuntu sources have moved to the /etc/apt/sources.list.d/ubuntu.sources +# file, which uses the deb822 format. Use deb822-formatted .sources files +# to manage package sources in the /etc/apt/sources.list.d/ directory. +# See the sources.list(5) manual page for details. +` + +var ubuntuSourceHeader = `## Ubuntu distribution repository +## +## The following settings can be adjusted to configure which packages to use from Ubuntu. +## Mirror your choices (except for URIs and Suites) in the security section below to +## ensure timely security updates. +## +## Types: Append deb-src to enable the fetching of source package. +## URIs: A URL to the repository (you may add multiple URLs) +## Suites: The following additional suites can be configured +## -updates - Major bug fix updates produced after the final release of the +## distribution. +## -backports - software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +## Components: Aside from main, the following components can be added to the list +## restricted - Software that may not be under a free license, or protected by patents. +## universe - Community maintained packages. +## Software from this repository is only maintained and supported by Canonical +## for machines with Ubuntu Pro subscriptions. Without Ubuntu Pro, the Ubuntu +## community provides best-effort security maintenance. +## multiverse - Community maintained of restricted. Software from this repository is +## ENTIRELY UNSUPPORTED by the Ubuntu team, and may not be under a free +## licence. Please satisfy yourself as to your rights to use the software. +## Also, please note that software in multiverse WILL NOT receive any +## review or updates from the Ubuntu security team. +## +## See the sources.list(5) manual page for further settings. +` + +var ubuntuSourceSecurityHeader = `## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +` + +// deb822SourcesList returns the content of /etc/apt/sources.list.d/ubuntu.sources +// to be used during the build process +func (i *ImageDefinition) Deb822BuildSourcesList() string { + return i.deb822SourcesList(false) +} + +// deb822SourcesList returns the content of /etc/apt/sources.list.d/ubuntu.sources +// for the target image +func (i *ImageDefinition) Deb822TargetSourcesList() string { + return i.deb822SourcesList(true) +} + +// deb822SourcesList returns the content of /etc/apt/sources.list.d/ubuntu.sources +// in the deb822 format. +// The target param defines if the generated sources list will be used in the target image. +func (i *ImageDefinition) deb822SourcesList(target bool) string { + pocket := i.Rootfs.Pocket + if target { + pocket = i.Customization.Pocket + } + pocket = strings.ToLower(pocket) + + ubuntuSources := ubuntuSourceHeader + generateDeb822Section( + i.Rootfs.Mirror, + i.Series, + i.Rootfs.Components, + pocket, + ) + + ubuntuSources += ubuntuSourceSecurityHeader + generateDeb822Section( + i.securityMirror(), + i.Series, + i.Rootfs.Components, + pocket, + ) + + return ubuntuSources } diff --git a/internal/imagedefinition/image_definition_test.go b/internal/imagedefinition/image_definition_test.go index 61e10366..d540a0da 100644 --- a/internal/imagedefinition/image_definition_test.go +++ b/internal/imagedefinition/image_definition_test.go @@ -12,6 +12,7 @@ import ( func TestGeneratePocketList(t *testing.T) { t.Parallel() + asserter := helper.Asserter{T: t} type args struct { series string components []string @@ -21,10 +22,10 @@ func TestGeneratePocketList(t *testing.T) { } testCases := []struct { - name string - imageDef ImageDefinition - args args - expectedPockets []string + name string + imageDef ImageDefinition + args args + expectedSourcesList string }{ { name: "release", @@ -35,7 +36,10 @@ func TestGeneratePocketList(t *testing.T) { securityMirror: "http://security.ubuntu.com/ubuntu/", pocket: "release", }, - expectedPockets: []string{"deb http://archive.ubuntu.com/ubuntu/ jammy main universe\n"}, + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy main universe +`, }, { name: "security", @@ -46,10 +50,11 @@ func TestGeneratePocketList(t *testing.T) { pocket: "security", securityMirror: "http://security.ubuntu.com/ubuntu/", }, - expectedPockets: []string{ - "deb http://archive.ubuntu.com/ubuntu/ jammy main\n", - "deb http://security.ubuntu.com/ubuntu/ jammy-security main\n", - }, + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy main +deb http://security.ubuntu.com/ubuntu/ jammy-security main +`, }, { name: "updates", @@ -60,11 +65,14 @@ func TestGeneratePocketList(t *testing.T) { securityMirror: "http://ports.ubuntu.com/", pocket: "updates", }, - expectedPockets: []string{ - "deb http://ports.ubuntu.com/ jammy main universe multiverse\n", - "deb http://ports.ubuntu.com/ jammy-security main universe multiverse\n", - "deb http://ports.ubuntu.com/ jammy-updates main universe multiverse\n", - }, + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://ports.ubuntu.com/ jammy main universe multiverse +deb http://ports.ubuntu.com/ jammy-security main universe multiverse +## Major bug fix updates produced after the final release of the +## distribution. +deb http://ports.ubuntu.com/ jammy-updates main universe multiverse +`, }, { name: "proposed", @@ -75,34 +83,28 @@ func TestGeneratePocketList(t *testing.T) { securityMirror: "http://security.ubuntu.com/ubuntu/", pocket: "proposed", }, - expectedPockets: []string{ - "deb http://archive.ubuntu.com/ubuntu/ jammy main universe multiverse restricted\n", - "deb http://security.ubuntu.com/ubuntu/ jammy-security main universe multiverse restricted\n", - "deb http://archive.ubuntu.com/ubuntu/ jammy-updates main universe multiverse restricted\n", - "deb http://archive.ubuntu.com/ubuntu/ jammy-proposed main universe multiverse restricted\n", - }, + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy main universe multiverse restricted +deb http://security.ubuntu.com/ubuntu/ jammy-security main universe multiverse restricted +## Major bug fix updates produced after the final release of the +## distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy-updates main universe multiverse restricted +deb http://archive.ubuntu.com/ubuntu/ jammy-proposed main universe multiverse restricted +`, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - pocketList := generatePocketList( + gotSourcesList := generateLegacySourcesList( tc.args.series, tc.args.components, tc.args.mirror, tc.args.securityMirror, tc.args.pocket, ) - for _, expectedPocket := range tc.expectedPockets { - found := false - for _, pocket := range pocketList { - if pocket == expectedPocket { - found = true - } - } - if !found { - t.Errorf("Expected %s in pockets list %s, but it was not", expectedPocket, pocketList) - } - } + + asserter.AssertEqual(tc.expectedSourcesList, gotSourcesList) }) } } @@ -204,12 +206,13 @@ func TestImageDefinition_SetDefaults(t *testing.T) { Seed: &Seed{ Vcs: helper.BoolPtr(true), }, - Tarball: &Tarball{}, - Components: []string{"main", "restricted"}, - Archive: "ubuntu", - Flavor: "ubuntu", - Mirror: "http://archive.ubuntu.com/ubuntu/", - Pocket: "release", + Tarball: &Tarball{}, + Components: []string{"main", "restricted"}, + Archive: "ubuntu", + Flavor: "ubuntu", + Mirror: "http://archive.ubuntu.com/ubuntu/", + Pocket: "release", + SourcesListDeb822: helper.BoolPtr(false), }, Customization: &Customization{ Components: []string{"main", "restricted", "universe"}, @@ -316,3 +319,105 @@ func TestImageDefinition_securityMirror(t *testing.T) { }) } } + +func Test_generateDeb822Section(t *testing.T) { + asserter := helper.Asserter{T: t} + type args struct { + mirror string + series string + components []string + pocket string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "release", + args: args{ + mirror: "http://archive.ubuntu.com/ubuntu/", + series: "jammy", + components: []string{"main", "restricted"}, + pocket: "release", + }, + want: `Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: jammy +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +`, + }, + { + name: "security", + args: args{ + mirror: "http://security.ubuntu.com/ubuntu/", + series: "jammy", + components: []string{"main", "restricted"}, + pocket: "security", + }, + want: `Types: deb +URIs: http://security.ubuntu.com/ubuntu/ +Suites: jammy-security +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +`, + }, + { + name: "proposed", + args: args{ + mirror: "http://archive.ubuntu.com/ubuntu/", + series: "jammy", + components: []string{"main", "restricted"}, + pocket: "proposed", + }, + want: `Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: jammy jammy-updates jammy-proposed +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +`, + }, + { + name: "updates", + args: args{ + mirror: "http://archive.ubuntu.com/ubuntu/", + series: "jammy", + components: []string{"main", "restricted"}, + pocket: "updates", + }, + want: `Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: jammy jammy-updates +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +`, + }, + { + name: "no pocket", + args: args{ + mirror: "http://archive.ubuntu.com/ubuntu/", + series: "jammy", + components: []string{"main", "restricted"}, + pocket: "", + }, + want: `Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := generateDeb822Section(tc.args.mirror, tc.args.series, tc.args.components, tc.args.pocket) + asserter.AssertEqual(tc.want, got) + }) + } +} diff --git a/internal/statemachine/classic.go b/internal/statemachine/classic.go index 6a8ba030..adaab0c2 100644 --- a/internal/statemachine/classic.go +++ b/internal/statemachine/classic.go @@ -92,12 +92,20 @@ func (stateMachine *StateMachine) parseImageDefinition() error { return err } + if imageDefinition.Rootfs.SourcesListDeb822 == nil { + fmt.Print("WARNING: rootfs.sources-list-deb822 was not set. Please explicitely set the format desired for sources list in your image definition.\n") + } + // populate the default values for imageDefinition if they were not provided in // the image definition YAML file if err := helperSetDefaults(&imageDefinition); err != nil { return err } + if !*imageDefinition.Rootfs.SourcesListDeb822 { + fmt.Print("WARNING: rootfs.sources-list-deb822 is set to false. The deprecated format will be used to manage sources list. Please if possible adopt the new format.\n") + } + // The official standard for YAML schemas states that they are an extension of // JSON schema draft 4. We therefore validate the decoded YAML against a JSON // schema. The workflow is as follows: diff --git a/internal/statemachine/classic_states.go b/internal/statemachine/classic_states.go index 8a841431..3469e283 100644 --- a/internal/statemachine/classic_states.go +++ b/internal/statemachine/classic_states.go @@ -176,8 +176,15 @@ func (stateMachine *StateMachine) createChroot() error { return fmt.Errorf("Error truncating resolv.conf: %s", err.Error()) } - // add any extra apt sources to /etc/apt/sources.list - return stateMachine.overwriteSourcesList(classicStateMachine.ImageDef.BuildPocketList()) + if *classicStateMachine.ImageDef.Rootfs.SourcesListDeb822 { + err := stateMachine.setDeb822SourcesList(classicStateMachine.ImageDef.Deb822BuildSourcesList()) + if err != nil { + return err + } + return stateMachine.setLegacySourcesList(imagedefinition.LegacySourcesListComment) + } + + return stateMachine.setLegacySourcesList(classicStateMachine.ImageDef.LegacyBuildSourcesList()) } // add PPAs to the apt sources list @@ -959,27 +966,58 @@ func (stateMachine *StateMachine) populateClassicRootfsContents() error { // is done, and before other manual customization to let users modify it. func (stateMachine *StateMachine) customizeSourcesList() error { classicStateMachine := stateMachine.parent.(*ClassicStateMachine) - return stateMachine.overwriteSourcesList(classicStateMachine.ImageDef.TargetPocketList()) + + if *classicStateMachine.ImageDef.Rootfs.SourcesListDeb822 { + err := stateMachine.setDeb822SourcesList(classicStateMachine.ImageDef.Deb822TargetSourcesList()) + if err != nil { + return err + } + return stateMachine.setLegacySourcesList(imagedefinition.LegacySourcesListComment) + } + + return stateMachine.setLegacySourcesList(classicStateMachine.ImageDef.LegacyTargetSourcesList()) } -// overwriteSourcesList replaces /etc/apt/sources.list with the given list of entries +// setLegacySourcesList replaces /etc/apt/sources.list with the given list of entries // This function will truncate the existing file. -func (stateMachine *StateMachine) overwriteSourcesList(aptSources []string) error { +func (stateMachine *StateMachine) setLegacySourcesList(aptSources string) error { sourcesList := filepath.Join(stateMachine.tempDirs.chroot, "etc", "apt", "sources.list") sourcesListFile, err := osOpenFile(sourcesList, os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { return fmt.Errorf("unable to open sources.list file: %w", err) } defer sourcesListFile.Close() - for _, aptSource := range aptSources { - _, err = sourcesListFile.WriteString(aptSource) - if err != nil { - return fmt.Errorf("unable to write apt sources: %w", err) - } + _, err = sourcesListFile.WriteString(aptSources) + if err != nil { + return fmt.Errorf("unable to write apt sources: %w", err) } return nil } +// setDeb822SourcesList replaces /etc/apt/sources.list.d/ubuntu.sources with the given content +// This function will truncate the existing file if any +func (stateMachine *StateMachine) setDeb822SourcesList(sourcesListContent string) error { + sourcesListDir := filepath.Join(stateMachine.tempDirs.chroot, "etc", "apt", "sources.list.d") + err := osMkdirAll(sourcesListDir, 0755) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("Error /etc/apt/sources.list.d directory: %s", err.Error()) + } + + sourcesList := filepath.Join(sourcesListDir, "ubuntu.sources") + f, err := osOpenFile(sourcesList, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("unable to open ubuntu.sources file: %w", err) + } + defer f.Close() + + _, err = f.WriteString(sourcesListContent) + if err != nil { + return fmt.Errorf("unable to write apt sources: %w", err) + } + + return nil +} + // fixFstab makes sure the fstab contains a valid entry for the root mount point func (stateMachine *StateMachine) fixFstab() error { classicStateMachine := stateMachine.parent.(*ClassicStateMachine) diff --git a/internal/statemachine/classic_test.go b/internal/statemachine/classic_test.go index 21609072..4005f3b8 100644 --- a/internal/statemachine/classic_test.go +++ b/internal/statemachine/classic_test.go @@ -2264,42 +2264,55 @@ func TestStateMachine_FailedPopulateClassicRootfsContents(t *testing.T) { // TestSateMachine_customizeSourcesList tests functionality of the customizeSourcesList state function func TestSateMachine_customizeSourcesList(t *testing.T) { testCases := []struct { - name string - existingSourcesList string - customization *imagedefinition.Customization - mockFuncs func() func() - expectedErr string - expectedSourcesList string + name string + deb822Format bool + existingSourcesList string + existingDeb822SourcesList string + customization *imagedefinition.Customization + mockFuncs func() func() + expectedErr string + expectedSourcesList string + expectedDeb822SourcesList string }{ { - name: "set default", + name: "set default sources.list", + deb822Format: false, existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", customization: &imagedefinition.Customization{}, - expectedSourcesList: `deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe `, }, { - name: "set less components", + name: "set less components sources.list", + deb822Format: false, existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", customization: &imagedefinition.Customization{ Components: []string{"main"}, }, - expectedSourcesList: `deb http://archive.ubuntu.com/ubuntu/ jammy main + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy main `, }, { - name: "set components and pocket", + name: "set components and pocket sources.list", + deb822Format: false, existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", customization: &imagedefinition.Customization{ Components: []string{"main"}, Pocket: "security", }, - expectedSourcesList: `deb http://archive.ubuntu.com/ubuntu/ jammy main + expectedSourcesList: `# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ jammy main deb http://security.ubuntu.com/ubuntu/ jammy-security main `, }, { name: "fail to write sources.list", + deb822Format: false, existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", customization: &imagedefinition.Customization{ Components: []string{"main"}, @@ -2318,6 +2331,123 @@ deb http://security.ubuntu.com/ubuntu/ jammy-security main return func() { osOpenFile = os.OpenFile } }, }, + { + name: "set default ubuntu.sources and commented sources.list", + deb822Format: true, + existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", + existingDeb822SourcesList: `Types: deb +URIs: http://archive.ubuntu.com/ +Suites: jammy +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +`, + customization: &imagedefinition.Customization{}, + expectedSourcesList: imagedefinition.LegacySourcesListComment, + expectedDeb822SourcesList: `## Ubuntu distribution repository +## +## The following settings can be adjusted to configure which packages to use from Ubuntu. +## Mirror your choices (except for URIs and Suites) in the security section below to +## ensure timely security updates. +## +## Types: Append deb-src to enable the fetching of source package. +## URIs: A URL to the repository (you may add multiple URLs) +## Suites: The following additional suites can be configured +## -updates - Major bug fix updates produced after the final release of the +## distribution. +## -backports - software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +## Components: Aside from main, the following components can be added to the list +## restricted - Software that may not be under a free license, or protected by patents. +## universe - Community maintained packages. +## Software from this repository is only maintained and supported by Canonical +## for machines with Ubuntu Pro subscriptions. Without Ubuntu Pro, the Ubuntu +## community provides best-effort security maintenance. +## multiverse - Community maintained of restricted. Software from this repository is +## ENTIRELY UNSUPPORTED by the Ubuntu team, and may not be under a free +## licence. Please satisfy yourself as to your rights to use the software. +## Also, please note that software in multiverse WILL NOT receive any +## review or updates from the Ubuntu security team. +## +## See the sources.list(5) manual page for further settings. +Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: jammy +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +Types: deb +URIs: http://security.ubuntu.com/ubuntu/ +Suites: jammy +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +`, + }, + { + name: "fail to write ubuntu.sources and commented sources.list", + deb822Format: true, + existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", + existingDeb822SourcesList: `Types: deb +URIs: http://archive.ubuntu.com/ +Suites: jammy +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +`, + customization: &imagedefinition.Customization{}, + expectedSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", + expectedDeb822SourcesList: `Types: deb +URIs: http://archive.ubuntu.com/ +Suites: jammy +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +`, + expectedErr: "unable to open ubuntu.sources file", + mockFuncs: func() func() { + mock := NewOSMock( + &osMockConf{ + OpenFileThreshold: 0, + }, + ) + + osOpenFile = mock.OpenFile + return func() { osOpenFile = os.OpenFile } + }, + }, + { + name: "fail to create sources.list.d", + deb822Format: true, + existingSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", + existingDeb822SourcesList: `Types: deb +URIs: http://archive.ubuntu.com/ +Suites: jammy +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +`, + customization: &imagedefinition.Customization{}, + expectedSourcesList: "deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted", + expectedDeb822SourcesList: `Types: deb +URIs: http://archive.ubuntu.com/ +Suites: jammy +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +`, + expectedErr: "Error /etc/apt/sources.list.d directory", + mockFuncs: func() func() { + mock := NewOSMock( + &osMockConf{ + MkdirAllThreshold: 0, + }, + ) + + osMkdirAll = mock.MkdirAll + return func() { osMkdirAll = os.MkdirAll } + }, + }, } for _, tc := range testCases { @@ -2330,9 +2460,11 @@ deb http://security.ubuntu.com/ubuntu/ jammy-security main stateMachine.commonFlags, stateMachine.stateMachineFlags = helper.InitCommonOpts() stateMachine.parent = &stateMachine stateMachine.ImageDef = imagedefinition.ImageDefinition{ - Architecture: getHostArch(), - Series: getHostSuite(), - Rootfs: &imagedefinition.Rootfs{}, + Architecture: getHostArch(), + Series: getHostSuite(), + Rootfs: &imagedefinition.Rootfs{ + SourcesListDeb822: helper.BoolPtr(tc.deb822Format), + }, Customization: tc.customization, } @@ -2344,14 +2476,18 @@ deb http://security.ubuntu.com/ubuntu/ jammy-security main t.Cleanup(func() { os.RemoveAll(stateMachine.stateMachineFlags.WorkDir) }) - err = os.MkdirAll(filepath.Join(stateMachine.tempDirs.chroot, "etc", "apt"), 0644) + err = os.MkdirAll(filepath.Join(stateMachine.tempDirs.chroot, "etc", "apt", "sources.list.d"), 0644) asserter.AssertErrNil(err, true) sourcesListPath := filepath.Join(stateMachine.tempDirs.chroot, "etc", "apt", "sources.list") + deb822SourcesListPath := filepath.Join(stateMachine.tempDirs.chroot, "etc", "apt", "sources.list.d", "ubuntu.sources") err = osWriteFile(sourcesListPath, []byte(tc.existingSourcesList), 0644) asserter.AssertErrNil(err, true) + err = osWriteFile(deb822SourcesListPath, []byte(tc.existingDeb822SourcesList), 0644) + asserter.AssertErrNil(err, true) + if tc.mockFuncs != nil { restoreMock := tc.mockFuncs() t.Cleanup(restoreMock) @@ -2365,10 +2501,13 @@ deb http://security.ubuntu.com/ubuntu/ jammy-security main sourcesListBytes, err := os.ReadFile(sourcesListPath) asserter.AssertErrNil(err, true) - if string(sourcesListBytes) != tc.expectedSourcesList { - t.Errorf("Expected sources.list content \"%s\", but got \"%s\"", - tc.expectedSourcesList, string(sourcesListBytes)) - } + asserter.AssertEqual(tc.expectedSourcesList, string(sourcesListBytes)) + + deb822SourcesListBytes, err := os.ReadFile(deb822SourcesListPath) + asserter.AssertErrNil(err, true) + + asserter.AssertEqual(tc.expectedDeb822SourcesList, string(deb822SourcesListBytes)) + }) } } @@ -2994,15 +3133,59 @@ func TestSuccessfulClassicRun(t *testing.T) { t.Errorf("Expected LANG=C.UTF-8 in %s, but got %s", localeFile, string(localeBytes)) } + // check if components and pocket correctly setup in /etc/apt/sources.list.d/ubuntu.sources + aptDeb822SourcesListBytes, err := os.ReadFile(filepath.Join(mountDir, "etc", "apt", "sources.list.d", "ubuntu.sources")) + asserter.AssertErrNil(err, true) + wantAptDeb822SourcesList := `## Ubuntu distribution repository +## +## The following settings can be adjusted to configure which packages to use from Ubuntu. +## Mirror your choices (except for URIs and Suites) in the security section below to +## ensure timely security updates. +## +## Types: Append deb-src to enable the fetching of source package. +## URIs: A URL to the repository (you may add multiple URLs) +## Suites: The following additional suites can be configured +## -updates - Major bug fix updates produced after the final release of the +## distribution. +## -backports - software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +## Components: Aside from main, the following components can be added to the list +## restricted - Software that may not be under a free license, or protected by patents. +## universe - Community maintained packages. +## Software from this repository is only maintained and supported by Canonical +## for machines with Ubuntu Pro subscriptions. Without Ubuntu Pro, the Ubuntu +## community provides best-effort security maintenance. +## multiverse - Community maintained of restricted. Software from this repository is +## ENTIRELY UNSUPPORTED by the Ubuntu team, and may not be under a free +## licence. Please satisfy yourself as to your rights to use the software. +## Also, please note that software in multiverse WILL NOT receive any +## review or updates from the Ubuntu security team. +## +## See the sources.list(5) manual page for further settings. +Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: jammy +Components: main universe restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +Types: deb +URIs: http://security.ubuntu.com/ubuntu/ +Suites: jammy +Components: main restricted +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +` + asserter.AssertEqual(wantAptDeb822SourcesList, string(aptDeb822SourcesListBytes)) + // check if components and pocket correctly setup in /etc/apt/sources.list aptSourcesListBytes, err := os.ReadFile(filepath.Join(mountDir, "etc", "apt", "sources.list")) asserter.AssertErrNil(err, true) - wantAptSourcesList := `deb http://archive.ubuntu.com/ubuntu/ jammy main universe restricted multiverse -deb http://security.ubuntu.com/ubuntu/ jammy-security main universe restricted multiverse -deb http://archive.ubuntu.com/ubuntu/ jammy-updates main universe restricted multiverse -deb http://archive.ubuntu.com/ubuntu/ jammy-proposed main universe restricted multiverse -` - asserter.AssertEqual(wantAptSourcesList, string(aptSourcesListBytes)) + asserter.AssertEqual(imagedefinition.LegacySourcesListComment, string(aptSourcesListBytes)) } @@ -3716,6 +3899,7 @@ func TestCreateChroot(t *testing.T) { t.Errorf("%s is not present in /etc/apt/sources.list", pocket) } } + } // TestFailedCreateChroot tests failure cases in createChroot @@ -3733,7 +3917,9 @@ func TestFailedCreateChroot(t *testing.T) { stateMachine.ImageDef = imagedefinition.ImageDefinition{ Architecture: getHostArch(), Series: getHostSuite(), - Rootfs: &imagedefinition.Rootfs{}, + Rootfs: &imagedefinition.Rootfs{ + SourcesListDeb822: helper.BoolPtr(false), + }, } err := stateMachine.makeTemporaryDirectories() diff --git a/internal/statemachine/testdata/image_definitions/test_amd64.yaml b/internal/statemachine/testdata/image_definitions/test_amd64.yaml index 48696a75..b8e92341 100644 --- a/internal/statemachine/testdata/image_definitions/test_amd64.yaml +++ b/internal/statemachine/testdata/image_definitions/test_amd64.yaml @@ -14,6 +14,7 @@ rootfs: - main - universe - restricted + sources-list-deb822: true seed: urls: - "git://git.launchpad.net/~ubuntu-core-dev/ubuntu-seeds/+git/" diff --git a/internal/statemachine/testdata/image_definitions/test_build_gadget.yaml b/internal/statemachine/testdata/image_definitions/test_build_gadget.yaml index 64d9d3c3..51c88ae2 100644 --- a/internal/statemachine/testdata/image_definitions/test_build_gadget.yaml +++ b/internal/statemachine/testdata/image_definitions/test_build_gadget.yaml @@ -10,6 +10,7 @@ gadget: branch: classic type: "git" rootfs: + sources-list-deb822: true seed: urls: - "https://git.launchpad.net/~ubuntu-core-dev/ubuntu-seeds/+git/" diff --git a/internal/statemachine/testdata/image_definitions/test_customization.yaml b/internal/statemachine/testdata/image_definitions/test_customization.yaml index 3f3dd5d6..d29f1f9e 100644 --- a/internal/statemachine/testdata/image_definitions/test_customization.yaml +++ b/internal/statemachine/testdata/image_definitions/test_customization.yaml @@ -10,6 +10,7 @@ gadget: branch: classic type: "git" rootfs: + sources-list-deb822: true seed: urls: - "https://git.launchpad.net/~ubuntu-core-dev/ubuntu-seeds/+git/" diff --git a/internal/statemachine/tests_helper_test.go b/internal/statemachine/tests_helper_test.go index 9bd53c49..1b729985 100644 --- a/internal/statemachine/tests_helper_test.go +++ b/internal/statemachine/tests_helper_test.go @@ -19,7 +19,8 @@ var basicImageDef = imagedefinition.ImageDefinition{ Architecture: getHostArch(), Series: getHostSuite(), Rootfs: &imagedefinition.Rootfs{ - Archive: "ubuntu", + Archive: "ubuntu", + SourcesListDeb822: helper.BoolPtr(false), }, Customization: &imagedefinition.Customization{}, } @@ -115,6 +116,7 @@ type osMockConf struct { RemoveThreshold uint TruncateThreshold uint OpenFileThreshold uint + MkdirAllThreshold uint } // osMock holds methods to easily mock functions from os and snapd/osutil packages @@ -128,6 +130,7 @@ type osMock struct { beforeRemoveFail uint beforeTruncateFail uint beforeOpenFileFail uint + beforeMkdirAllFail uint } func (o *osMock) CopySpecialFile(path, dest string) error { @@ -175,6 +178,15 @@ func (o *osMock) OpenFile(name string, flag int, perm os.FileMode) (*os.File, er return &os.File{}, nil } +func (o *osMock) MkdirAll(path string, perm os.FileMode) error { + if o.beforeOpenFileFail >= o.conf.OpenFileThreshold { + return fmt.Errorf("OpenFile fail") + } + o.beforeMkdirAllFail++ + + return nil +} + func NewOSMock(conf *osMockConf) *osMock { return &osMock{conf: conf} }