From 084c05f420ed84b86b29e2bb174abaf371936e04 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 25 Apr 2019 14:07:53 -0700 Subject: [PATCH 001/877] *: start project --- base/base.go | 6 +++++ config/common.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++ distro/distro.go | 6 +++++ go.mod | 10 ++++++++ go.sum | 9 +++++++ internal/main.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 155 insertions(+) create mode 100644 base/base.go create mode 100644 config/common.go create mode 100644 distro/distro.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/main.go diff --git a/base/base.go b/base/base.go new file mode 100644 index 000000000..074751160 --- /dev/null +++ b/base/base.go @@ -0,0 +1,6 @@ +package base + +type Base struct { + Field1 int `yaml:"field1"` + Field2 int `yaml:"field2"` +} diff --git a/config/common.go b/config/common.go new file mode 100644 index 000000000..4916b9802 --- /dev/null +++ b/config/common.go @@ -0,0 +1,61 @@ +package config + +import ( + "errors" + "fmt" + + yaml "github.com/ajeddeloh/yaml" + "github.com/coreos/go-semver/semver" +) + +var ( + ErrNoVariant = errors.New("Error parsing variant. Variant must be specified") + ErrInvalidVersion = errors.New("Error parsing version. Version must be a valid semver") + + registry = map[string]translator{} +) + +func getTranslator(variant string, version semver.Version) (translator, error) { + t, ok := registry[fmt.Sprintf("%s+%s", variant, version.String())] + if !ok { + return nil, fmt.Errorf("No translator exists for variant %s with version %s", variant, version.String()) + } + return t, nil +} + +type TranslateOptions struct { + Pretty bool + Strict bool +} + +type Common struct { + Version string `yaml:"version"` + Variant string `yaml:"variant"` +} + +type translator func([]byte, TranslateOptions) ([]byte, error) + +// Translate wraps all of the actual translate functions in a switch that determines the correct one to call +func Translate(input []byte, options TranslateOptions) ([]byte, error) { + // first determine version. This will ignore most fields, so don't use strict + ver := Common{} + if err := yaml.Unmarshal(input, &ver); err != nil { + return nil, fmt.Errorf("Error unmarshaling yaml: %v", err) + } + + if ver.Variant == "" { + return nil, ErrNoVariant + } + + tmp, err := semver.NewVersion(ver.Version) + if err != nil { + return nil, ErrInvalidVersion + } + version := *tmp + + translator, err := getTranslator(ver.Variant, version) + if err != nil { + } + + return translator(input, options) +} diff --git a/distro/distro.go b/distro/distro.go new file mode 100644 index 000000000..3c6c104be --- /dev/null +++ b/distro/distro.go @@ -0,0 +1,6 @@ +package distro + +type Distro struct { + DField1 int + DField2 int +} diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..55519b536 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/ajeddeloh/fcct + +go 1.12 + +require ( + github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 + github.com/coreos/go-semver v0.3.0 + github.com/coreos/ignition v0.32.0 + github.com/go-yaml/yaml v2.1.0+incompatible // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..1f7485632 --- /dev/null +++ b/go.sum @@ -0,0 +1,9 @@ +github.com/ajeddeloh/ignition v0.6.0 h1:UmrrpeVWIB4lBHf2atn/wZbJI7sYLBUvE8aA5B6Wn6s= +github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 h1:5exOSffWnMuW0z7StqHxz6IFVV082C6rxw9/eR2U7r8= +github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/ignition v0.32.0 h1:uMFA1tWr7B9kq2Ku4Bmo61G4gUm/vHsOuognuYaS0yg= +github.com/coreos/ignition v0.32.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= +github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= diff --git a/internal/main.go b/internal/main.go new file mode 100644 index 000000000..be79bb272 --- /dev/null +++ b/internal/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + + "github.com/ajeddeloh/fcct/config" +) + +func fail(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, format, args...) + os.Exit(1) +} + +func main() { + var ( + input string + output string + ) + options := config.TranslateOptions{} + flag.BoolVar(&options.Strict, "strict", false, "fail on any warning") + flag.BoolVar(&options.Pretty, "pretty", false, "output formatted json") + flag.StringVar(&input, "input", "", "read from input file instead of stdin") + flag.StringVar(&output, "output", "", "write to output file instead of stdout") + + flag.Parse() + + var infile *os.File = os.Stdin + var outfile *os.File = os.Stdout + if input != "" { + var err error + infile, err = os.Open(input) + if err != nil { + fail("failed to open %s: %v", input, err) + } + defer infile.Close() + } + + dataIn, err := ioutil.ReadAll(infile) + if err != nil { + fail("failed to read %s: %v", infile.Name(), err) + } + + dataOut, err := config.Translate(dataIn, options) + if err != nil { + fail("Error translating config: %v", err) + } + + if output != "" { + var err error + outfile, err = os.Open(output) + if err != nil { + fail("failed to open %s: %v", output, err) + } + defer outfile.Close() + } + + if _, err := outfile.Write(dataOut); err != nil { + fail("Failed to write config to %s: %v", outfile.Name(), err) + } +} From bc088112eea97118d958256e340251467bf54bde Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 25 Apr 2019 14:29:17 -0700 Subject: [PATCH 002/877] readme: add readme --- README | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 README diff --git a/README b/README new file mode 100644 index 000000000..0e5563843 --- /dev/null +++ b/README @@ -0,0 +1,32 @@ +Fedora CoreOS Config Transpiler + +Still a work in progress. + +Each config spec is composed of a base and distro component. The base components +roughly mirror the Ignition spec and are distro agnostic. The distro compoents +contain sugar for common configuration of the host (e.g. etcd) and are not +distro-independent. + +Each distro and base component are versioned independently with each new +version getting it's own package. These versions are not exposed to the user. + +Each fcos config version has it's own version which is independent of the +versions of the base and distro components that compose it. However a major +or minor bump of either component necesitates a corresponding bump in the fcos +config version. + +internal/ + main, non-exported code + +base/ + Contains distro-agnostic code. Each package here targets only one Ignition + spec verions. + +distro/ + Contains distro-specific code. Each package here can target multiple Ignition + versions if it makes sense. + +config/ + Contains definitions combining an entry in base and an entry in distro. Also + contains the top level Translate() function that determines which version to + parse and emit. From 6d57f1f3f155dfaef2bcbfebc31e35476bfee519 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 26 Apr 2019 15:44:15 -0700 Subject: [PATCH 003/877] *: Implement skeleton Implement skeleton of the code without any of the actual translation logic. Does nothing right now. --- base/base.go | 6 -- base/v0_1/schema.go | 188 +++++++++++++++++++++++++++++++++++ base/v0_1/translate.go | 9 ++ config/common.go | 23 ++++- config/fcos.go | 32 ++++++ distro/distro.go | 6 -- distro/fcos_0_1/schema.go | 5 + distro/fcos_0_1/translate.go | 9 ++ go.mod | 7 +- go.sum | 26 +++++ 10 files changed, 295 insertions(+), 16 deletions(-) delete mode 100644 base/base.go create mode 100644 base/v0_1/schema.go create mode 100644 base/v0_1/translate.go create mode 100644 config/fcos.go delete mode 100644 distro/distro.go create mode 100644 distro/fcos_0_1/schema.go create mode 100644 distro/fcos_0_1/translate.go diff --git a/base/base.go b/base/base.go deleted file mode 100644 index 074751160..000000000 --- a/base/base.go +++ /dev/null @@ -1,6 +0,0 @@ -package base - -type Base struct { - Field1 int `yaml:"field1"` - Field2 int `yaml:"field2"` -} diff --git a/base/v0_1/schema.go b/base/v0_1/schema.go new file mode 100644 index 000000000..7771f4ddc --- /dev/null +++ b/base/v0_1/schema.go @@ -0,0 +1,188 @@ +package v0_1 + +type CaReference struct { + Source string `yaml:"source"` + Verification Verification `yaml:"verification"` +} + +type Config struct { + Ignition Ignition `yaml:"ignition"` + Passwd Passwd `yaml:"passwd"` + Storage Storage `yaml:"storage"` + Systemd Systemd `yaml:"systemd"` +} + +type ConfigReference struct { + Source *string `yaml:"source"` + Verification Verification `yaml:"verification"` +} + +type Device string + +type Directory struct { + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Mode *int `yaml:"mode"` +} + +type Disk struct { + Device string `yaml:"device"` + Partitions []Partition `yaml:"partitions"` + WipeTable *bool `yaml:"wipeTable"` +} + +type Dropin struct { + Contents *string `yaml:"contents"` + Name string `yaml:"name"` +} + +type File struct { + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Append []FileContents `yaml:"append"` + Contents FileContents `yaml:"contents"` + Mode *int `yaml:"mode"` +} + +type FileContents struct { + Compression *string `yaml:"compression"` + Source *string `yaml:"source"` + Verification Verification `yaml:"verification"` +} + +type Filesystem struct { + Device string `yaml:"device"` + Format *string `yaml:"format"` + Label *string `yaml:"label"` + Options []FilesystemOption `yaml:"options"` + Path *string `yaml:"path"` + UUID *string `yaml:"uuid"` + WipeFilesystem *bool `yaml:"wipeFilesystem"` +} + +type FilesystemOption string + +type Group string + +type Ignition struct { + Config IgnitionConfig `yaml:"config"` + Security Security `yaml:"security"` + Timeouts Timeouts `yaml:"timeouts"` +} + +type IgnitionConfig struct { + Merge []ConfigReference `yaml:"merge"` + Replace ConfigReference `yaml:"replace"` +} + +type Link struct { + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Hard *bool `yaml:"hard"` + Target string `yaml:"target"` +} + +type NodeGroup struct { + ID *int `yaml:"id"` + Name *string `yaml:"name"` +} + +type NodeUser struct { + ID *int `yaml:"id"` + Name *string `yaml:"name"` +} + +type Partition struct { + GUID *string `yaml:"guid"` + Label *string `yaml:"label"` + Number int `yaml:"number"` + ShouldExist *bool `yaml:"shouldExist"` + SizeMiB *int `yaml:"sizeMiB"` + StartMiB *int `yaml:"startMiB"` + TypeGUID *string `yaml:"typeGuid"` + WipePartitionEntry *bool `yaml:"wipePartitionEntry"` +} + +type Passwd struct { + Groups []PasswdGroup `yaml:"groups"` + Users []PasswdUser `yaml:"users"` +} + +type PasswdGroup struct { + Gid *int `yaml:"gid"` + Name string `yaml:"name"` + PasswordHash *string `yaml:"passwordHash"` + System *bool `yaml:"system"` +} + +type PasswdUser struct { + Gecos *string `yaml:"gecos"` + Groups []Group `yaml:"groups"` + HomeDir *string `yaml:"homeDir"` + Name string `yaml:"name"` + NoCreateHome *bool `yaml:"noCreateHome"` + NoLogInit *bool `yaml:"noLogInit"` + NoUserGroup *bool `yaml:"noUserGroup"` + PasswordHash *string `yaml:"passwordHash"` + PrimaryGroup *string `yaml:"primaryGroup"` + SSHAuthorizedKeys []SSHAuthorizedKey `yaml:"sshAuthorizedKeys"` + Shell *string `yaml:"shell"` + System *bool `yaml:"system"` + UID *int `yaml:"uid"` +} + +type Raid struct { + Devices []Device `yaml:"devices"` + Level string `yaml:"level"` + Name string `yaml:"name"` + Options []RaidOption `yaml:"options"` + Spares *int `yaml:"spares"` +} + +type RaidOption string + +type SSHAuthorizedKey string + +type Security struct { + TLS TLS `yaml:"tls"` +} + +type Storage struct { + Directories []Directory `yaml:"directories"` + Disks []Disk `yaml:"disks"` + Files []File `yaml:"files"` + Filesystems []Filesystem `yaml:"filesystems"` + Links []Link `yaml:"links"` + Raid []Raid `yaml:"raid"` +} + +type Systemd struct { + Units []Unit `yaml:"units"` +} + +type TLS struct { + CertificateAuthorities []CaReference `yaml:"certificateAuthorities"` +} + +type Timeouts struct { + HTTPResponseHeaders *int `yaml:"httpResponseHeaders"` + HTTPTotal *int `yaml:"httpTotal"` +} + +type Unit struct { + Contents *string `yaml:"contents"` + Dropins []Dropin `yaml:"dropins"` + Enabled *bool `yaml:"enabled"` + Mask *bool `yaml:"mask"` + Name string `yaml:"name"` +} + +type Verification struct { + Hash *string `yaml:"hash"` +} diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go new file mode 100644 index 000000000..16cd10bfd --- /dev/null +++ b/base/v0_1/translate.go @@ -0,0 +1,9 @@ +package v0_1 + +import ( + "github.com/coreos/ignition/v2/config/v3_0/types" +) + +func (c Config) ToIgn3_0() (types.Config, error) { + return types.Config{}, nil +} diff --git a/config/common.go b/config/common.go index 4916b9802..a0657b518 100644 --- a/config/common.go +++ b/config/common.go @@ -4,7 +4,8 @@ import ( "errors" "fmt" - yaml "github.com/ajeddeloh/yaml" + json "github.com/ajeddeloh/go-json" + "github.com/go-yaml/yaml" "github.com/coreos/go-semver/semver" ) @@ -12,7 +13,9 @@ var ( ErrNoVariant = errors.New("Error parsing variant. Variant must be specified") ErrInvalidVersion = errors.New("Error parsing version. Version must be a valid semver") - registry = map[string]translator{} + registry = map[string]translator{ + "fcos+1.0.0": TranslateFcos0_1, + } ) func getTranslator(variant string, version semver.Version) (translator, error) { @@ -55,7 +58,23 @@ func Translate(input []byte, options TranslateOptions) ([]byte, error) { translator, err := getTranslator(ver.Variant, version) if err != nil { + return nil, err } return translator(input, options) } + +// Misc helpers +func unmarshal(data []byte, to interface{}, strict bool) error { + if strict { + return yaml.UnmarshalStrict(data, to) + } + return yaml.Unmarshal(data, to) +} + +func marshal(from interface{}, pretty bool) ([]byte, error) { + if pretty { + return json.MarshalIndent(from, "", " ") + } + return json.Marshal(from) +} diff --git a/config/fcos.go b/config/fcos.go new file mode 100644 index 000000000..7f3bc5807 --- /dev/null +++ b/config/fcos.go @@ -0,0 +1,32 @@ +package config + +import ( + base_0_1 "github.com/ajeddeloh/fcct/base/v0_1" + "github.com/ajeddeloh/fcct/distro/fcos_0_1" +) + +type FcosConfig0_1 struct { + Common `yaml:",inline"` + base_0_1.Config `yaml:",inline"` + fcos_0_1.Fcos `yaml:",inline"` +} + +func dumbMerge(a, b interface{}) { + // pass +} + +func TranslateFcos0_1(input []byte, options TranslateOptions) ([]byte, error) { + cfg := FcosConfig0_1{} + + if err := unmarshal(input, &cfg, options.Strict); err != nil { + return nil, err + } + + base, _ := cfg.Config.ToIgn3_0() + distro, _ := cfg.Fcos.ToIgn3_0() + // do a dumb merge, these should not conflict and if they do the user should fix them + dumbMerge(&base, distro) + + // TODO validation + return marshal(base, options.Pretty) +} diff --git a/distro/distro.go b/distro/distro.go deleted file mode 100644 index 3c6c104be..000000000 --- a/distro/distro.go +++ /dev/null @@ -1,6 +0,0 @@ -package distro - -type Distro struct { - DField1 int - DField2 int -} diff --git a/distro/fcos_0_1/schema.go b/distro/fcos_0_1/schema.go new file mode 100644 index 000000000..d366a1314 --- /dev/null +++ b/distro/fcos_0_1/schema.go @@ -0,0 +1,5 @@ +package fcos_0_1 + +type Fcos struct { + Foobar string `yaml:"foobar"` +} diff --git a/distro/fcos_0_1/translate.go b/distro/fcos_0_1/translate.go new file mode 100644 index 000000000..151a1e84a --- /dev/null +++ b/distro/fcos_0_1/translate.go @@ -0,0 +1,9 @@ +package fcos_0_1 + +import ( + "github.com/coreos/ignition/v2/config/v3_0/types" +) + +func (f Fcos) ToIgn3_0() (types.Config, error) { + return types.Config{}, nil +} diff --git a/go.mod b/go.mod index 55519b536..028fc366f 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,11 @@ module github.com/ajeddeloh/fcct go 1.12 require ( + github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 github.com/coreos/go-semver v0.3.0 - github.com/coreos/ignition v0.32.0 - github.com/go-yaml/yaml v2.1.0+incompatible // indirect + github.com/coreos/ignition/v2 v2.0.0-alpha + github.com/go-yaml/yaml v2.1.0+incompatible ) + +replace github.com/coreos/ignition/v2 => ../ignition diff --git a/go.sum b/go.sum index 1f7485632..6840ac966 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,35 @@ +github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU= +github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/ignition v0.6.0 h1:UmrrpeVWIB4lBHf2atn/wZbJI7sYLBUvE8aA5B6Wn6s= github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 h1:5exOSffWnMuW0z7StqHxz6IFVV082C6rxw9/eR2U7r8= github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= +github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/ignition v0.32.0 h1:uMFA1tWr7B9kq2Ku4Bmo61G4gUm/vHsOuognuYaS0yg= github.com/coreos/ignition v0.32.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= +github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= +golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 839f90947d3381b9d5535109bce49ff17d357fd4 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 6 May 2019 11:04:31 -0700 Subject: [PATCH 004/877] distro/fcos: mv fcos_0_1 -> fcos/v0_1 Mirror whats happening in the base packages --- config/fcos.go | 2 +- distro/{fcos_0_1 => fcos/v0_1}/schema.go | 0 distro/{fcos_0_1 => fcos/v0_1}/translate.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename distro/{fcos_0_1 => fcos/v0_1}/schema.go (100%) rename distro/{fcos_0_1 => fcos/v0_1}/translate.go (100%) diff --git a/config/fcos.go b/config/fcos.go index 7f3bc5807..1817762a8 100644 --- a/config/fcos.go +++ b/config/fcos.go @@ -2,7 +2,7 @@ package config import ( base_0_1 "github.com/ajeddeloh/fcct/base/v0_1" - "github.com/ajeddeloh/fcct/distro/fcos_0_1" + fcos_0_1 "github.com/ajeddeloh/fcct/distro/fcos/v0_1" ) type FcosConfig0_1 struct { diff --git a/distro/fcos_0_1/schema.go b/distro/fcos/v0_1/schema.go similarity index 100% rename from distro/fcos_0_1/schema.go rename to distro/fcos/v0_1/schema.go diff --git a/distro/fcos_0_1/translate.go b/distro/fcos/v0_1/translate.go similarity index 100% rename from distro/fcos_0_1/translate.go rename to distro/fcos/v0_1/translate.go From a00a79be4e3898aab58cf3766a5b05dd0a470f67 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 6 May 2019 13:23:39 -0700 Subject: [PATCH 005/877] go.*: switch to upstream project --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 028fc366f..3a22f9bd4 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,6 @@ require ( github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 github.com/coreos/go-semver v0.3.0 - github.com/coreos/ignition/v2 v2.0.0-alpha + github.com/coreos/ignition/v2 v2.0.0-beta github.com/go-yaml/yaml v2.1.0+incompatible ) - -replace github.com/coreos/ignition/v2 => ../ignition diff --git a/go.sum b/go.sum index 6840ac966..07d56800c 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/ignition v0.32.0 h1:uMFA1tWr7B9kq2Ku4Bmo61G4gUm/vHsOuognuYaS0yg= github.com/coreos/ignition v0.32.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= +github.com/coreos/ignition/v2 v2.0.0-beta h1:QU7mbrAWip+dp9RQ8tZQ2x1t7uWQEcTJBd6f6+TlB78= +github.com/coreos/ignition/v2 v2.0.0-beta/go.mod h1:VzUffEQLl900URFrEZtOdDi0IlBwJrYJXh/jeCB9nbc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= From 9191640f30f73051e379547efd351b1d28386d19 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 6 May 2019 13:24:11 -0700 Subject: [PATCH 006/877] *: validate intermediate and final configs Ensure that the configs generated from both the distro dependent and independent bits are valid, and that after merging them the resultant config is valid. Additionally, fix the translating code to generate valid configs --- base/v0_1/translate.go | 6 +++++- config/fcos.go | 37 ++++++++++++++++++++++++++--------- distro/fcos/v0_1/translate.go | 6 +++++- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go index 16cd10bfd..c799b28bf 100644 --- a/base/v0_1/translate.go +++ b/base/v0_1/translate.go @@ -5,5 +5,9 @@ import ( ) func (c Config) ToIgn3_0() (types.Config, error) { - return types.Config{}, nil + return types.Config{ + Ignition: types.Ignition{ + Version: "3.0.0", + }, + }, nil } diff --git a/config/fcos.go b/config/fcos.go index 1817762a8..2f7ba6d32 100644 --- a/config/fcos.go +++ b/config/fcos.go @@ -1,8 +1,19 @@ package config import ( + "errors" + "fmt" + "reflect" + base_0_1 "github.com/ajeddeloh/fcct/base/v0_1" fcos_0_1 "github.com/ajeddeloh/fcct/distro/fcos/v0_1" + + "github.com/coreos/ignition/v2/config/v3_0" + "github.com/coreos/ignition/v2/config/validate" +) + +var ( + ErrInvalidConfig = errors.New("config generated was invalid") ) type FcosConfig0_1 struct { @@ -11,10 +22,6 @@ type FcosConfig0_1 struct { fcos_0_1.Fcos `yaml:",inline"` } -func dumbMerge(a, b interface{}) { - // pass -} - func TranslateFcos0_1(input []byte, options TranslateOptions) ([]byte, error) { cfg := FcosConfig0_1{} @@ -22,11 +29,23 @@ func TranslateFcos0_1(input []byte, options TranslateOptions) ([]byte, error) { return nil, err } - base, _ := cfg.Config.ToIgn3_0() - distro, _ := cfg.Fcos.ToIgn3_0() - // do a dumb merge, these should not conflict and if they do the user should fix them - dumbMerge(&base, distro) + base, err := cfg.Config.ToIgn3_0() + if err != nil { + return nil, err + } + + distro, err := cfg.Fcos.ToIgn3_0() + if err != nil { + return nil, err + } + + final := v3_0.Merge(distro, base) + r := validate.ValidateWithoutSource(reflect.ValueOf(final)) + fmt.Println(r.String()) + if r.IsFatal() { + return nil, ErrInvalidConfig + } // TODO validation - return marshal(base, options.Pretty) + return marshal(final, options.Pretty) } diff --git a/distro/fcos/v0_1/translate.go b/distro/fcos/v0_1/translate.go index 151a1e84a..3d27061da 100644 --- a/distro/fcos/v0_1/translate.go +++ b/distro/fcos/v0_1/translate.go @@ -5,5 +5,9 @@ import ( ) func (f Fcos) ToIgn3_0() (types.Config, error) { - return types.Config{}, nil + return types.Config{ + Ignition: types.Ignition{ + Version: "3.0.0", + }, + }, nil } From 5edbdd9ff7a37ddf85ebb4c2c28fb09c7f51bf58 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 6 May 2019 15:16:03 -0700 Subject: [PATCH 007/877] base/v0_1: add translation Implement the actual meat of the translation. This doesn't have the nicities of inlining files yet, but should have eveything else. --- base/v0_1/translate.go | 56 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go index c799b28bf..773926f77 100644 --- a/base/v0_1/translate.go +++ b/base/v0_1/translate.go @@ -1,13 +1,59 @@ package v0_1 import ( + "github.com/coreos/ignition/v2/config/translate" "github.com/coreos/ignition/v2/config/v3_0/types" ) func (c Config) ToIgn3_0() (types.Config, error) { - return types.Config{ - Ignition: types.Ignition{ - Version: "3.0.0", - }, - }, nil + ret := types.Config{} + tr := translate.NewTranslator() + tr.AddCustomTranslator(translateIgnition) + tr.AddCustomTranslator(translateFile) + tr.AddCustomTranslator(translateDirectory) + tr.AddCustomTranslator(translateLink) + tr.Translate(&c, &ret) + return ret, nil +} + +func translateIgnition(from Ignition) (to types.Ignition) { + tr := translate.NewTranslator() + to.Version = "3.0.0" + tr.Translate(&from.Config, &to.Config) + tr.Translate(&from.Security, &to.Security) + tr.Translate(&from.Timeouts, &to.Timeouts) + return +} + +func translateFile(from File) (to types.File) { + tr := translate.NewTranslator() + tr.Translate(&from.Group, &to.Group) + tr.Translate(&from.User, &to.User) + tr.Translate(&from.Append, &to.Append) + tr.Translate(&from.Contents, &to.Contents) + to.Overwrite = from.Overwrite + to.Path = from.Path + to.Mode = from.Mode + return +} + +func translateDirectory(from Directory) (to types.Directory) { + tr := translate.NewTranslator() + tr.Translate(&from.Group, &to.Group) + tr.Translate(&from.User, &to.User) + to.Overwrite = from.Overwrite + to.Path = from.Path + to.Mode = from.Mode + return +} + +func translateLink(from Link) (to types.Link) { + tr := translate.NewTranslator() + tr.Translate(&from.Group, &to.Group) + tr.Translate(&from.User, &to.User) + to.Target = from.Target + to.Hard = from.Hard + to.Overwrite = from.Overwrite + to.Path = from.Path + return } From a4f60a7141836684cdf056355d3ba3811fbb2c44 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 9 May 2019 13:39:32 -0700 Subject: [PATCH 008/877] *: break out common and versioned code Split up config/ into some subpackages so each version can be its own package. Break out common code into it's own pacakge to avoid import cycles. Use the new go yaml v3 library instead of my fork since upstream now has the needed functionality. --- config/common/common.go | 32 +++++++++++++++++++++++++++ config/{common.go => config.go} | 38 +++++++-------------------------- config/{ => v1_0}/fcos.go | 11 +++++----- go.mod | 5 ++--- go.sum | 14 ++++++------ internal/main.go | 3 ++- 6 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 config/common/common.go rename config/{common.go => config.go} (62%) rename config/{ => v1_0}/fcos.go (73%) diff --git a/config/common/common.go b/config/common/common.go new file mode 100644 index 000000000..2e6cb36da --- /dev/null +++ b/config/common/common.go @@ -0,0 +1,32 @@ +package common + +import ( + "bytes" + "encoding/json" + + "gopkg.in/yaml.v3" +) + +type TranslateOptions struct { + Pretty bool + Strict bool +} + +type Common struct { + Version string `yaml:"version"` + Variant string `yaml:"variant"` +} + +// Misc helpers +func Unmarshal(data []byte, to interface{}, strict bool) error { + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(strict) + return dec.Decode(to) +} + +func Marshal(from interface{}, pretty bool) ([]byte, error) { + if pretty { + return json.MarshalIndent(from, "", " ") + } + return json.Marshal(from) +} diff --git a/config/common.go b/config/config.go similarity index 62% rename from config/common.go rename to config/config.go index a0657b518..246cd215f 100644 --- a/config/common.go +++ b/config/config.go @@ -4,9 +4,11 @@ import ( "errors" "fmt" - json "github.com/ajeddeloh/go-json" - "github.com/go-yaml/yaml" + "github.com/ajeddeloh/fcct/config/common" + "github.com/ajeddeloh/fcct/config/v1_0" + "github.com/coreos/go-semver/semver" + "gopkg.in/yaml.v3" ) var ( @@ -14,7 +16,7 @@ var ( ErrInvalidVersion = errors.New("Error parsing version. Version must be a valid semver") registry = map[string]translator{ - "fcos+1.0.0": TranslateFcos0_1, + "fcos+1.0.0": v1_0.TranslateFcos0_1, } ) @@ -26,22 +28,12 @@ func getTranslator(variant string, version semver.Version) (translator, error) { return t, nil } -type TranslateOptions struct { - Pretty bool - Strict bool -} - -type Common struct { - Version string `yaml:"version"` - Variant string `yaml:"variant"` -} - -type translator func([]byte, TranslateOptions) ([]byte, error) +type translator func([]byte, common.TranslateOptions) ([]byte, error) // Translate wraps all of the actual translate functions in a switch that determines the correct one to call -func Translate(input []byte, options TranslateOptions) ([]byte, error) { +func Translate(input []byte, options common.TranslateOptions) ([]byte, error) { // first determine version. This will ignore most fields, so don't use strict - ver := Common{} + ver := common.Common{} if err := yaml.Unmarshal(input, &ver); err != nil { return nil, fmt.Errorf("Error unmarshaling yaml: %v", err) } @@ -64,17 +56,3 @@ func Translate(input []byte, options TranslateOptions) ([]byte, error) { return translator(input, options) } -// Misc helpers -func unmarshal(data []byte, to interface{}, strict bool) error { - if strict { - return yaml.UnmarshalStrict(data, to) - } - return yaml.Unmarshal(data, to) -} - -func marshal(from interface{}, pretty bool) ([]byte, error) { - if pretty { - return json.MarshalIndent(from, "", " ") - } - return json.Marshal(from) -} diff --git a/config/fcos.go b/config/v1_0/fcos.go similarity index 73% rename from config/fcos.go rename to config/v1_0/fcos.go index 2f7ba6d32..af2036ebf 100644 --- a/config/fcos.go +++ b/config/v1_0/fcos.go @@ -1,10 +1,11 @@ -package config +package v1_0 import ( "errors" "fmt" "reflect" + "github.com/ajeddeloh/fcct/config/common" base_0_1 "github.com/ajeddeloh/fcct/base/v0_1" fcos_0_1 "github.com/ajeddeloh/fcct/distro/fcos/v0_1" @@ -17,15 +18,15 @@ var ( ) type FcosConfig0_1 struct { - Common `yaml:",inline"` + common.Common `yaml:",inline"` base_0_1.Config `yaml:",inline"` fcos_0_1.Fcos `yaml:",inline"` } -func TranslateFcos0_1(input []byte, options TranslateOptions) ([]byte, error) { +func TranslateFcos0_1(input []byte, options common.TranslateOptions) ([]byte, error) { cfg := FcosConfig0_1{} - if err := unmarshal(input, &cfg, options.Strict); err != nil { + if err := common.Unmarshal(input, &cfg, options.Strict); err != nil { return nil, err } @@ -47,5 +48,5 @@ func TranslateFcos0_1(input []byte, options TranslateOptions) ([]byte, error) { } // TODO validation - return marshal(final, options.Pretty) + return common.Marshal(final, options.Pretty) } diff --git a/go.mod b/go.mod index 3a22f9bd4..1cbe6fd1d 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module github.com/ajeddeloh/fcct go 1.12 require ( - github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 - github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 + github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 // indirect github.com/coreos/go-semver v0.3.0 github.com/coreos/ignition/v2 v2.0.0-beta - github.com/go-yaml/yaml v2.1.0+incompatible + gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae ) diff --git a/go.sum b/go.sum index 07d56800c..c106c910f 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,27 @@ github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajeddeloh/ignition v0.6.0 h1:UmrrpeVWIB4lBHf2atn/wZbJI7sYLBUvE8aA5B6Wn6s= -github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269 h1:5exOSffWnMuW0z7StqHxz6IFVV082C6rxw9/eR2U7r8= -github.com/ajeddeloh/yaml v0.0.0-20141224210557-6b16a5714269/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/ignition v0.32.0 h1:uMFA1tWr7B9kq2Ku4Bmo61G4gUm/vHsOuognuYaS0yg= -github.com/coreos/ignition v0.32.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= github.com/coreos/ignition/v2 v2.0.0-beta h1:QU7mbrAWip+dp9RQ8tZQ2x1t7uWQEcTJBd6f6+TlB78= github.com/coreos/ignition/v2 v2.0.0-beta/go.mod h1:VzUffEQLl900URFrEZtOdDi0IlBwJrYJXh/jeCB9nbc= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= @@ -33,5 +29,9 @@ github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9o github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae h1:ehhBuCxzgQEGk38YjhFv/97fMIc2JGHZAhAWMmEjmu0= +gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/main.go b/internal/main.go index be79bb272..d7762ac84 100644 --- a/internal/main.go +++ b/internal/main.go @@ -7,6 +7,7 @@ import ( "os" "github.com/ajeddeloh/fcct/config" + "github.com/ajeddeloh/fcct/config/common" ) func fail(format string, args ...interface{}) { @@ -19,7 +20,7 @@ func main() { input string output string ) - options := config.TranslateOptions{} + options := common.TranslateOptions{} flag.BoolVar(&options.Strict, "strict", false, "fail on any warning") flag.BoolVar(&options.Pretty, "pretty", false, "output formatted json") flag.StringVar(&input, "input", "", "read from input file instead of stdin") From 06b09f8c73dcba526c5bbcdc40af84b9601eb62e Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 9 May 2019 14:10:06 -0700 Subject: [PATCH 009/877] config, readme: more refactoring, update readme Split up some of the fcos translate function to be more testable. Also update the readme to reflect the current state of the repo. Now that each version is in it's own package, drop the versions in the struct names. --- README | 13 +++++++++++-- config/config.go | 2 +- config/v1_0/fcos.go | 36 +++++++++++++++++++++++++----------- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/README b/README index 0e5563843..684ff5be5 100644 --- a/README +++ b/README @@ -27,6 +27,15 @@ distro/ versions if it makes sense. config/ - Contains definitions combining an entry in base and an entry in distro. Also - contains the top level Translate() function that determines which version to + Contains the top level Translate() function that determines which version to parse and emit. + +config/common/ + Contains the common bits and functions for all spec versions. This means the + (un)marshaling helpers and the version+variant struct to be included in every + user facing spec + +config/vX_Y/ + Contains user facing definitions of the spec. Each is composed by combining a + base and distro package with the common version+variant. Each of the defines + their own translate function to be registered in the config/ package. diff --git a/config/config.go b/config/config.go index 246cd215f..0dd54cac8 100644 --- a/config/config.go +++ b/config/config.go @@ -16,7 +16,7 @@ var ( ErrInvalidVersion = errors.New("Error parsing version. Version must be a valid semver") registry = map[string]translator{ - "fcos+1.0.0": v1_0.TranslateFcos0_1, + "fcos+1.0.0": v1_0.TranslateBytes, } ) diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index af2036ebf..d595789a8 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -10,6 +10,7 @@ import ( fcos_0_1 "github.com/ajeddeloh/fcct/distro/fcos/v0_1" "github.com/coreos/ignition/v2/config/v3_0" + "github.com/coreos/ignition/v2/config/v3_0/types" "github.com/coreos/ignition/v2/config/validate" ) @@ -17,36 +18,49 @@ var ( ErrInvalidConfig = errors.New("config generated was invalid") ) -type FcosConfig0_1 struct { +type Config struct { common.Common `yaml:",inline"` base_0_1.Config `yaml:",inline"` fcos_0_1.Fcos `yaml:",inline"` } -func TranslateFcos0_1(input []byte, options common.TranslateOptions) ([]byte, error) { - cfg := FcosConfig0_1{} - - if err := common.Unmarshal(input, &cfg, options.Strict); err != nil { - return nil, err +func (c Config) Translate() (types.Config, error) { + base, err := c.Config.ToIgn3_0() + if err != nil { + return types.Config{}, err } - base, err := cfg.Config.ToIgn3_0() + distro, err := c.Fcos.ToIgn3_0() if err != nil { + return types.Config{}, err + } + + return v3_0.Merge(distro, base), nil +} + +func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, error) { + cfg := Config{} + + if err := common.Unmarshal(input, &cfg, options.Strict); err != nil { return nil, err } + r := validate.ValidateWithoutSource(reflect.ValueOf(cfg)) + if r.IsFatal() { + fmt.Println(r.String()) + return nil, ErrInvalidConfig + } - distro, err := cfg.Fcos.ToIgn3_0() + final, err := cfg.Translate() if err != nil { return nil, err } - final := v3_0.Merge(distro, base) - r := validate.ValidateWithoutSource(reflect.ValueOf(final)) + r.Merge(validate.ValidateWithoutSource(reflect.ValueOf(final))) fmt.Println(r.String()) + if r.IsFatal() { return nil, ErrInvalidConfig } - // TODO validation return common.Marshal(final, options.Pretty) } From e3cfea32f28099b16f7d0c92e6a190afc1933db0 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 23 May 2019 14:06:00 -0700 Subject: [PATCH 010/877] base/v0_1: switch to snake_case --- base/v0_1/schema.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/base/v0_1/schema.go b/base/v0_1/schema.go index 7771f4ddc..7190238ab 100644 --- a/base/v0_1/schema.go +++ b/base/v0_1/schema.go @@ -30,7 +30,7 @@ type Directory struct { type Disk struct { Device string `yaml:"device"` Partitions []Partition `yaml:"partitions"` - WipeTable *bool `yaml:"wipeTable"` + WipeTable *bool `yaml:"wipe_table"` } type Dropin struct { @@ -61,7 +61,7 @@ type Filesystem struct { Options []FilesystemOption `yaml:"options"` Path *string `yaml:"path"` UUID *string `yaml:"uuid"` - WipeFilesystem *bool `yaml:"wipeFilesystem"` + WipeFilesystem *bool `yaml:"wipe_filesystem"` } type FilesystemOption string @@ -102,11 +102,11 @@ type Partition struct { GUID *string `yaml:"guid"` Label *string `yaml:"label"` Number int `yaml:"number"` - ShouldExist *bool `yaml:"shouldExist"` - SizeMiB *int `yaml:"sizeMiB"` - StartMiB *int `yaml:"startMiB"` - TypeGUID *string `yaml:"typeGuid"` - WipePartitionEntry *bool `yaml:"wipePartitionEntry"` + ShouldExist *bool `yaml:"should_exist"` + SizeMiB *int `yaml:"size_mib"` + StartMiB *int `yaml:"start_mib"` + TypeGUID *string `yaml:"type_guid"` + WipePartitionEntry *bool `yaml:"wipe_partition_entry"` } type Passwd struct { @@ -117,21 +117,21 @@ type Passwd struct { type PasswdGroup struct { Gid *int `yaml:"gid"` Name string `yaml:"name"` - PasswordHash *string `yaml:"passwordHash"` + PasswordHash *string `yaml:"password_hash"` System *bool `yaml:"system"` } type PasswdUser struct { Gecos *string `yaml:"gecos"` Groups []Group `yaml:"groups"` - HomeDir *string `yaml:"homeDir"` + HomeDir *string `yaml:"home_dir"` Name string `yaml:"name"` - NoCreateHome *bool `yaml:"noCreateHome"` - NoLogInit *bool `yaml:"noLogInit"` - NoUserGroup *bool `yaml:"noUserGroup"` - PasswordHash *string `yaml:"passwordHash"` - PrimaryGroup *string `yaml:"primaryGroup"` - SSHAuthorizedKeys []SSHAuthorizedKey `yaml:"sshAuthorizedKeys"` + NoCreateHome *bool `yaml:"no_create_home"` + NoLogInit *bool `yaml:"no_log_init"` + NoUserGroup *bool `yaml:"no_user_group"` + PasswordHash *string `yaml:"password_hash"` + PrimaryGroup *string `yaml:"primary_group"` + SSHAuthorizedKeys []SSHAuthorizedKey `yaml:"ssh_authorized_keys"` Shell *string `yaml:"shell"` System *bool `yaml:"system"` UID *int `yaml:"uid"` @@ -167,12 +167,12 @@ type Systemd struct { } type TLS struct { - CertificateAuthorities []CaReference `yaml:"certificateAuthorities"` + CertificateAuthorities []CaReference `yaml:"certificate_authorities"` } type Timeouts struct { - HTTPResponseHeaders *int `yaml:"httpResponseHeaders"` - HTTPTotal *int `yaml:"httpTotal"` + HTTPResponseHeaders *int `yaml:"http_response_headers"` + HTTPTotal *int `yaml:"http_total"` } type Unit struct { From b4d04b2ef72cb7f431a4bb68aa6f5ba8e10b4913 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 5 Jun 2019 13:32:02 -0700 Subject: [PATCH 011/877] config: plumb line/col through to validation Translate the context trees from snake_case to camelCase so Ignition's type's validation can use the context tree from the yaml. --- config/common/common.go | 47 +++++++++++++++++++++++++++++++++++++++-- config/v1_0/fcos.go | 22 +++++++++++++++---- go.mod | 6 ++++-- go.sum | 19 ++++++++++++++--- 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/config/common/common.go b/config/common/common.go index 2e6cb36da..79a1b9816 100644 --- a/config/common/common.go +++ b/config/common/common.go @@ -3,7 +3,10 @@ package common import ( "bytes" "encoding/json" + "strings" + vyaml "github.com/coreos/vcontext/yaml" + "github.com/coreos/vcontext/tree" "gopkg.in/yaml.v3" ) @@ -18,10 +21,13 @@ type Common struct { } // Misc helpers -func Unmarshal(data []byte, to interface{}, strict bool) error { +func Unmarshal(data []byte, to interface{}, strict bool) (tree.Node, error) { dec := yaml.NewDecoder(bytes.NewReader(data)) dec.KnownFields(strict) - return dec.Decode(to) + if err := dec.Decode(to); err != nil { + return nil, err + } + return vyaml.UnmarshalToContext(data) } func Marshal(from interface{}, pretty bool) ([]byte, error) { @@ -30,3 +36,40 @@ func Marshal(from interface{}, pretty bool) ([]byte, error) { } return json.Marshal(from) } + +func camel(in string) string { + words := strings.Split(in, "_") + for i, word := range words[1:] { + words[i+1] = strings.Title(word) + } + return strings.Join(words, "") +} + +func ToCamelCase(t tree.Node) tree.Node { + switch n := t.(type) { + case tree.MapNode: + m := tree.MapNode{ + Children: make(map[string]tree.Node, len(n.Children)), + Keys: make(map[string]tree.Leaf, len(n.Keys)), + Marker: n.Marker, + } + for k, v := range n.Children { + m.Children[camel(k)] = ToCamelCase(v) + } + for k, v := range n.Keys { + m.Keys[camel(k)] = v + } + return m + case tree.SliceNode: + s := tree.SliceNode{ + Children: make([]tree.Node, 0, len(n.Children)), + Marker: n.Marker, + } + for _, v := range n.Children { + s.Children = append(s.Children, ToCamelCase(v)) + } + return s + default: // leaf + return t + } +} diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index d595789a8..5ec5fc281 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -11,7 +11,11 @@ import ( "github.com/coreos/ignition/v2/config/v3_0" "github.com/coreos/ignition/v2/config/v3_0/types" - "github.com/coreos/ignition/v2/config/validate" + ignvalidate "github.com/coreos/ignition/v2/config/validate" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" + "github.com/coreos/vcontext/validate" + ) var ( @@ -41,10 +45,17 @@ func (c Config) Translate() (types.Config, error) { func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, error) { cfg := Config{} - if err := common.Unmarshal(input, &cfg, options.Strict); err != nil { + contextTree, err := common.Unmarshal(input, &cfg, options.Strict) + if err != nil { return nil, err } - r := validate.ValidateWithoutSource(reflect.ValueOf(cfg)) + + r := validate.Validate(cfg, "yaml") + unusedKeyCheck := func(v reflect.Value, c path.ContextPath) report.Report { + return ignvalidate.ValidateUnusedKeys(v, c, contextTree) + } + r.Merge(validate.ValidateCustom(cfg, "yaml", unusedKeyCheck)) + r.Correlate(contextTree) if r.IsFatal() { fmt.Println(r.String()) return nil, ErrInvalidConfig @@ -55,7 +66,10 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, erro return nil, err } - r.Merge(validate.ValidateWithoutSource(reflect.ValueOf(final))) + translatedTree := common.ToCamelCase(contextTree) + second := validate.Validate(final, "json") + second.Correlate(translatedTree) + r.Merge(second) fmt.Println(r.String()) if r.IsFatal() { diff --git a/go.mod b/go.mod index 1cbe6fd1d..527a71ead 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module github.com/ajeddeloh/fcct go 1.12 require ( - github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 // indirect github.com/coreos/go-semver v0.3.0 - github.com/coreos/ignition/v2 v2.0.0-beta + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/ignition/v2 v2.0.0 + github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a + github.com/davecgh/go-spew v1.1.1 // indirect gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae ) diff --git a/go.sum b/go.sum index c106c910f..89d68b24b 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -6,15 +5,27 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/ignition/v2 v2.0.0-beta h1:QU7mbrAWip+dp9RQ8tZQ2x1t7uWQEcTJBd6f6+TlB78= -github.com/coreos/ignition/v2 v2.0.0-beta/go.mod h1:VzUffEQLl900URFrEZtOdDi0IlBwJrYJXh/jeCB9nbc= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/ignition/v2 v2.0.0 h1:2nIU6rQIh2bCSF0PNJKuNNa23L13G031CoxbI4GFojc= +github.com/coreos/ignition/v2 v2.0.0/go.mod h1:EJP9Gk/u21BPwWa5nT6yYbm5mO0u8JQAAzFqaxgH7Fg= +github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= +github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a h1:AdSIR3t2/3oreDLKnsCk8Lu5L3+OKlt9Yu7equBivmE= +github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -31,6 +42,8 @@ golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae h1:ehhBuCxzgQEGk38YjhFv/97fMIc2JGHZAhAWMmEjmu0= From 06c29251017dbfb96b6e725f79fa031ef400a438 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 5 Jun 2019 13:33:57 -0700 Subject: [PATCH 012/877] distro: remove junk left over from testing Remove extraneous field left over from testing when the project was very young. --- distro/fcos/v0_1/schema.go | 1 - 1 file changed, 1 deletion(-) diff --git a/distro/fcos/v0_1/schema.go b/distro/fcos/v0_1/schema.go index d366a1314..442711f5b 100644 --- a/distro/fcos/v0_1/schema.go +++ b/distro/fcos/v0_1/schema.go @@ -1,5 +1,4 @@ package fcos_0_1 type Fcos struct { - Foobar string `yaml:"foobar"` } From 8ef947544e01bfd8d8b5cab6a0ff77a2752c5e36 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 5 Jun 2019 14:43:41 -0700 Subject: [PATCH 013/877] *: fix gofmt --- config/common/common.go | 8 ++++---- config/config.go | 1 - config/v1_0/fcos.go | 3 +-- internal/main.go | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/config/common/common.go b/config/common/common.go index 79a1b9816..1b5bc412d 100644 --- a/config/common/common.go +++ b/config/common/common.go @@ -5,8 +5,8 @@ import ( "encoding/json" "strings" - vyaml "github.com/coreos/vcontext/yaml" "github.com/coreos/vcontext/tree" + vyaml "github.com/coreos/vcontext/yaml" "gopkg.in/yaml.v3" ) @@ -49,9 +49,9 @@ func ToCamelCase(t tree.Node) tree.Node { switch n := t.(type) { case tree.MapNode: m := tree.MapNode{ - Children: make(map[string]tree.Node, len(n.Children)), - Keys: make(map[string]tree.Leaf, len(n.Keys)), - Marker: n.Marker, + Children: make(map[string]tree.Node, len(n.Children)), + Keys: make(map[string]tree.Leaf, len(n.Keys)), + Marker: n.Marker, } for k, v := range n.Children { m.Children[camel(k)] = ToCamelCase(v) diff --git a/config/config.go b/config/config.go index 0dd54cac8..2d430a229 100644 --- a/config/config.go +++ b/config/config.go @@ -55,4 +55,3 @@ func Translate(input []byte, options common.TranslateOptions) ([]byte, error) { return translator(input, options) } - diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index 5ec5fc281..693543e9c 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -5,8 +5,8 @@ import ( "fmt" "reflect" - "github.com/ajeddeloh/fcct/config/common" base_0_1 "github.com/ajeddeloh/fcct/base/v0_1" + "github.com/ajeddeloh/fcct/config/common" fcos_0_1 "github.com/ajeddeloh/fcct/distro/fcos/v0_1" "github.com/coreos/ignition/v2/config/v3_0" @@ -15,7 +15,6 @@ import ( "github.com/coreos/vcontext/path" "github.com/coreos/vcontext/report" "github.com/coreos/vcontext/validate" - ) var ( diff --git a/internal/main.go b/internal/main.go index d7762ac84..fce84d690 100644 --- a/internal/main.go +++ b/internal/main.go @@ -17,8 +17,8 @@ func fail(format string, args ...interface{}) { func main() { var ( - input string - output string + input string + output string ) options := common.TranslateOptions{} flag.BoolVar(&options.Strict, "strict", false, "fail on any warning") From c14f2e2a639467beeb0f484a6e3e0abf22ae440a Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 5 Jun 2019 14:44:00 -0700 Subject: [PATCH 014/877] tests: add test script and tests Add tests for: - translating structs from v1 fccl to v3 Ignition - snake_case to camelCase function - snake_case to camelCase conversion of context trees --- base/v0_1/translate_test.go | 264 +++++++++++++++++++++++++++++++++++ config/common/common_test.go | 129 +++++++++++++++++ test | 12 ++ 3 files changed, 405 insertions(+) create mode 100644 base/v0_1/translate_test.go create mode 100644 config/common/common_test.go create mode 100755 test diff --git a/base/v0_1/translate_test.go b/base/v0_1/translate_test.go new file mode 100644 index 000000000..d1e8cbca4 --- /dev/null +++ b/base/v0_1/translate_test.go @@ -0,0 +1,264 @@ +package v0_1 + +import ( + "reflect" + "testing" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/ignition/v2/config/v3_0/types" +) + +// Most of this is covered by the Ignition translator generic tests, so just test the custom bits + +func TestTranslateFile(t *testing.T) { + tests := []struct { + in File + out types.File + }{ + { + File{}, + types.File{}, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + File{ + Path: "/foo", + Group: NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Mode: util.IntToPtr(420), + Append: []FileContents{ + { + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + { + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + Overwrite: util.BoolToPtr(true), + Contents: FileContents{ + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + types.File{ + Node: types.Node{ + Path: "/foo", + Group: types.NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: types.NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + }, + FileEmbedded1: types.FileEmbedded1{ + Mode: util.IntToPtr(420), + Append: []types.FileContents{ + { + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: types.Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + { + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: types.Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + Contents: types.FileContents{ + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: types.Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + }, + }, + } + + for i, test := range tests { + actual := translateFile(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +func TestTranslateDirectory(t *testing.T) { + tests := []struct { + in Directory + out types.Directory + }{ + { + Directory{}, + types.Directory{}, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + Directory{ + Path: "/foo", + Group: NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Mode: util.IntToPtr(420), + Overwrite: util.BoolToPtr(true), + }, + types.Directory{ + Node: types.Node{ + Path: "/foo", + Group: types.NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: types.NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + }, + DirectoryEmbedded1: types.DirectoryEmbedded1{ + Mode: util.IntToPtr(420), + }, + }, + }, + } + + for i, test := range tests { + actual := translateDirectory(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +func TestTranslateLink(t *testing.T) { + tests := []struct { + in Link + out types.Link + }{ + { + Link{}, + types.Link{}, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + Link{ + Path: "/foo", + Group: NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + Target: "/bar", + Hard: util.BoolToPtr(false), + }, + types.Link{ + Node: types.Node{ + Path: "/foo", + Group: types.NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: types.NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + }, + LinkEmbedded1: types.LinkEmbedded1{ + Target: "/bar", + Hard: util.BoolToPtr(false), + }, + }, + }, + } + + for i, test := range tests { + actual := translateLink(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +func TestTranslateIgnition(t *testing.T) { + tests := []struct { + in Ignition + out types.Ignition + }{ + { + Ignition{}, + types.Ignition{ + Version: "3.0.0", + }, + }, + } + for i, test := range tests { + actual := translateIgnition(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +func TestToIgn3_0(t *testing.T) { + tests := []struct { + in Config + out types.Config + }{ + { + Config{}, + types.Config{ + Ignition: types.Ignition{ + Version: "3.0.0", + }, + }, + }, + } + for i, test := range tests { + actual, err := test.in.ToIgn3_0() + if err != nil { + t.Errorf("#%d: got error: %v", i, err) + } + + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} diff --git a/config/common/common_test.go b/config/common/common_test.go new file mode 100644 index 000000000..6fdbe4d57 --- /dev/null +++ b/config/common/common_test.go @@ -0,0 +1,129 @@ +package common + +import ( + "reflect" + "testing" + + "github.com/coreos/vcontext/tree" +) + +func TestCamel(t *testing.T) { + tests := []struct { + in string + out string + }{ + {}, + { + "foo", + "foo", + }, + { + "snake_case", + "snakeCase", + }, + { + "long_snake_case", + "longSnakeCase", + }, + { + "camelAlready", + "camelAlready", + }, + } + + for i, test := range tests { + if camel(test.in) != test.out { + t.Errorf("#%d: expected %q got %q", i, test.out, camel(test.in)) + } + } +} + +func TestToCamelCase(t *testing.T) { + tests := []struct { + in tree.Node + out tree.Node + }{ + {}, + { + tree.Leaf{ + Marker: tree.MarkerFromIndices(1, 2), + }, + tree.Leaf{ + Marker: tree.MarkerFromIndices(1, 2), + }, + }, + { + tree.MapNode{ + Marker: tree.MarkerFromIndices(1, 2), + Children: map[string]tree.Node{ + "foo_bar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + Keys: map[string]tree.Leaf{ + "foo_bar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + }, + tree.MapNode{ + Marker: tree.MarkerFromIndices(1, 2), + Children: map[string]tree.Node{ + "fooBar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + Keys: map[string]tree.Leaf{ + "fooBar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + }, + }, + { + tree.SliceNode{ + Marker: tree.MarkerFromIndices(5, 6), + Children: []tree.Node{ + tree.MapNode{ + Marker: tree.MarkerFromIndices(1, 2), + Children: map[string]tree.Node{ + "foo_bar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + Keys: map[string]tree.Leaf{ + "foo_bar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + }, + }, + }, + tree.SliceNode{ + Marker: tree.MarkerFromIndices(5, 6), + Children: []tree.Node{ + tree.MapNode{ + Marker: tree.MarkerFromIndices(1, 2), + Children: map[string]tree.Node{ + "fooBar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + Keys: map[string]tree.Leaf{ + "fooBar": tree.Leaf{ + tree.MarkerFromIndices(3, 4), + }, + }, + }, + }, + }, + }, + } + + for i, test := range tests { + actual := ToCamelCase(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} diff --git a/test b/test new file mode 100755 index 000000000..b13eb9d02 --- /dev/null +++ b/test @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +export GO111MODULE=on + +echo "checking gofmt" +res=$(gofmt -d .) +echo "$res" +test -z "$res" + +echo "running tests" +go test ./... -cover From 81e30c4633ab9f550a72d1b4c9ea94f99fb996dc Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 5 Jun 2019 15:53:54 -0700 Subject: [PATCH 015/877] *: add ASL 2.0 license --- LICENSE | 202 ++++++++++++++++++++++++++++++++++ base/v0_1/schema.go | 14 +++ base/v0_1/translate.go | 14 +++ base/v0_1/translate_test.go | 14 +++ config/common/common.go | 14 +++ config/common/common_test.go | 14 +++ config/config.go | 14 +++ config/v1_0/fcos.go | 14 +++ distro/fcos/v0_1/schema.go | 14 +++ distro/fcos/v0_1/translate.go | 14 +++ internal/main.go | 14 +++ 11 files changed, 342 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..e06d20818 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/base/v0_1/schema.go b/base/v0_1/schema.go index 7190238ab..b57d9927e 100644 --- a/base/v0_1/schema.go +++ b/base/v0_1/schema.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package v0_1 type CaReference struct { diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go index 773926f77..0139d44e1 100644 --- a/base/v0_1/translate.go +++ b/base/v0_1/translate.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package v0_1 import ( diff --git a/base/v0_1/translate_test.go b/base/v0_1/translate_test.go index d1e8cbca4..9771bfd25 100644 --- a/base/v0_1/translate_test.go +++ b/base/v0_1/translate_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package v0_1 import ( diff --git a/config/common/common.go b/config/common/common.go index 1b5bc412d..46cc68a0b 100644 --- a/config/common/common.go +++ b/config/common/common.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package common import ( diff --git a/config/common/common_test.go b/config/common/common_test.go index 6fdbe4d57..a6de2539d 100644 --- a/config/common/common_test.go +++ b/config/common/common_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package common import ( diff --git a/config/config.go b/config/config.go index 2d430a229..7293c4440 100644 --- a/config/config.go +++ b/config/config.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package config import ( diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index 693543e9c..edfc49832 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package v1_0 import ( diff --git a/distro/fcos/v0_1/schema.go b/distro/fcos/v0_1/schema.go index 442711f5b..ed0e0946d 100644 --- a/distro/fcos/v0_1/schema.go +++ b/distro/fcos/v0_1/schema.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package fcos_0_1 type Fcos struct { diff --git a/distro/fcos/v0_1/translate.go b/distro/fcos/v0_1/translate.go index 3d27061da..0f6188ee5 100644 --- a/distro/fcos/v0_1/translate.go +++ b/distro/fcos/v0_1/translate.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package fcos_0_1 import ( diff --git a/internal/main.go b/internal/main.go index fce84d690..f9481b562 100644 --- a/internal/main.go +++ b/internal/main.go @@ -1,3 +1,17 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + package main import ( From 930bdc38d86b8f7b84ccd7995890c739040084d8 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 6 Jun 2019 09:46:02 -0700 Subject: [PATCH 016/877] fixup! tests: add test script and tests --- base/v0_1/translate_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/v0_1/translate_test.go b/base/v0_1/translate_test.go index 9771bfd25..0a6f2d038 100644 --- a/base/v0_1/translate_test.go +++ b/base/v0_1/translate_test.go @@ -24,6 +24,7 @@ import ( // Most of this is covered by the Ignition translator generic tests, so just test the custom bits +// TestTranslateFile tests translating the ct storage.files.[i] entries to ignition storage.files.[i] entires. func TestTranslateFile(t *testing.T) { tests := []struct { in File @@ -123,6 +124,7 @@ func TestTranslateFile(t *testing.T) { } } +// TestTranslateDirectory tests translating the ct storage.directories.[i] entries to ignition storage.directories.[i] entires. func TestTranslateDirectory(t *testing.T) { tests := []struct { in Directory @@ -176,6 +178,7 @@ func TestTranslateDirectory(t *testing.T) { } } +// TestTranslateLink tests translating the ct storage.links.[i] entries to ignition storage.links.[i] entires. func TestTranslateLink(t *testing.T) { tests := []struct { in Link @@ -231,6 +234,8 @@ func TestTranslateLink(t *testing.T) { } } +// TestTranslateIgnition tests translating the ct config.ignition to the ignition config.ignition section. +// It ensure that the version is set as well. func TestTranslateIgnition(t *testing.T) { tests := []struct { in Ignition @@ -251,6 +256,8 @@ func TestTranslateIgnition(t *testing.T) { } } +// TestToIgn3_0 tests the config.ToIgn3_0 function ensuring it will generate a valid config even when empty. Not much else is +// tested since it uses the Ignition translation code which has it's own set of tests. func TestToIgn3_0(t *testing.T) { tests := []struct { in Config From ed77912229d1c7f35c3deb1e035bb6911c764041 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 6 Jun 2019 09:47:27 -0700 Subject: [PATCH 017/877] fixup! base/v0_1: add translation --- base/v0_1/translate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go index 0139d44e1..a461598dc 100644 --- a/base/v0_1/translate.go +++ b/base/v0_1/translate.go @@ -32,7 +32,7 @@ func (c Config) ToIgn3_0() (types.Config, error) { func translateIgnition(from Ignition) (to types.Ignition) { tr := translate.NewTranslator() - to.Version = "3.0.0" + to.Version = types.MaxVersion.String() tr.Translate(&from.Config, &to.Config) tr.Translate(&from.Security, &to.Security) tr.Translate(&from.Timeouts, &to.Timeouts) From 5c43e57b9e609b10a7b3ee578fac8b2d34f259db Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 6 Jun 2019 09:48:24 -0700 Subject: [PATCH 018/877] fixup! config: plumb line/col through to validation --- config/common/common.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/common/common.go b/config/common/common.go index 46cc68a0b..44b068645 100644 --- a/config/common/common.go +++ b/config/common/common.go @@ -35,6 +35,9 @@ type Common struct { } // Misc helpers + +// Unmarshal unmarshals the data to "to" and also returns a context tree for the source. If strict +// is set it errors out on unused keys. func Unmarshal(data []byte, to interface{}, strict bool) (tree.Node, error) { dec := yaml.NewDecoder(bytes.NewReader(data)) dec.KnownFields(strict) @@ -44,6 +47,7 @@ func Unmarshal(data []byte, to interface{}, strict bool) (tree.Node, error) { return vyaml.UnmarshalToContext(data) } +// Marshal is a wrapper for marshaling to json with or without pretty-printing the output func Marshal(from interface{}, pretty bool) ([]byte, error) { if pretty { return json.MarshalIndent(from, "", " ") @@ -51,6 +55,7 @@ func Marshal(from interface{}, pretty bool) ([]byte, error) { return json.Marshal(from) } +// camel takes a snake_case string and converting it to camelCase func camel(in string) string { words := strings.Split(in, "_") for i, word := range words[1:] { @@ -59,6 +64,7 @@ func camel(in string) string { return strings.Join(words, "") } +// ToCamelCase converts the keys in a context tree from snake_case to camelCase func ToCamelCase(t tree.Node) tree.Node { switch n := t.(type) { case tree.MapNode: From 03f6d11670c55ff19e806f849992d7530dbe93ae Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 8 Jul 2019 13:09:07 -0700 Subject: [PATCH 019/877] base: add sugar for inlining files Add a field storage.files.contents.inline in addition to the existing "source" field. "inline" is mutually exclusive with "source". --- base/v0_1/schema.go | 1 + base/v0_1/translate.go | 18 +++++++++ base/v0_1/translate_test.go | 4 +- base/v0_1/validate.go | 33 ++++++++++++++++ base/v0_1/validate_test.go | 79 +++++++++++++++++++++++++++++++++++++ go.mod | 1 + 6 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 base/v0_1/validate.go create mode 100644 base/v0_1/validate_test.go diff --git a/base/v0_1/schema.go b/base/v0_1/schema.go index b57d9927e..c1a70bcb9 100644 --- a/base/v0_1/schema.go +++ b/base/v0_1/schema.go @@ -65,6 +65,7 @@ type File struct { type FileContents struct { Compression *string `yaml:"compression"` Source *string `yaml:"source"` + Inline *string `yaml:"inline"` // Added, not in ignition spec Verification Verification `yaml:"verification"` } diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go index a461598dc..c48f25fb7 100644 --- a/base/v0_1/translate.go +++ b/base/v0_1/translate.go @@ -15,8 +15,11 @@ package v0_1 import ( + "net/url" + "github.com/coreos/ignition/v2/config/translate" "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/vincent-petithory/dataurl" ) func (c Config) ToIgn3_0() (types.Config, error) { @@ -41,6 +44,7 @@ func translateIgnition(from Ignition) (to types.Ignition) { func translateFile(from File) (to types.File) { tr := translate.NewTranslator() + tr.AddCustomTranslator(translateFileContents) tr.Translate(&from.Group, &to.Group) tr.Translate(&from.User, &to.User) tr.Translate(&from.Append, &to.Append) @@ -51,6 +55,20 @@ func translateFile(from File) (to types.File) { return } +func translateFileContents(from FileContents) (to types.FileContents) { + to.Source = from.Source + to.Compression = from.Compression + to.Verification.Hash = from.Verification.Hash + if from.Inline != nil { + src := (&url.URL{ + Scheme: "data", + Opaque: "," + dataurl.EscapeString(*from.Inline), + }).String() + to.Source = &src + } + return +} + func translateDirectory(from Directory) (to types.Directory) { tr := translate.NewTranslator() tr.Translate(&from.Group, &to.Group) diff --git a/base/v0_1/translate_test.go b/base/v0_1/translate_test.go index 0a6f2d038..c44b6806a 100644 --- a/base/v0_1/translate_test.go +++ b/base/v0_1/translate_test.go @@ -57,7 +57,7 @@ func TestTranslateFile(t *testing.T) { }, }, { - Source: util.StrToPtr("http://example/com"), + Inline: util.StrToPtr("hello"), Compression: util.StrToPtr("gzip"), Verification: Verification{ Hash: util.StrToPtr("this isn't validated"), @@ -97,7 +97,7 @@ func TestTranslateFile(t *testing.T) { }, }, { - Source: util.StrToPtr("http://example/com"), + Source: util.StrToPtr("data:,hello"), Compression: util.StrToPtr("gzip"), Verification: types.Verification{ Hash: util.StrToPtr("this isn't validated"), diff --git a/base/v0_1/validate.go b/base/v0_1/validate.go new file mode 100644 index 000000000..c8e6942aa --- /dev/null +++ b/base/v0_1/validate.go @@ -0,0 +1,33 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_1 + +import ( + "errors" + + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +var ( + ErrInlineAndSource = errors.New("inline cannot be specified if source is specified") +) + +func (f FileContents) Validate(c path.ContextPath) (r report.Report) { + if f.Inline != nil && f.Source != nil { + r.AddOnError(c.Append("inline"), ErrInlineAndSource) + } + return +} diff --git a/base/v0_1/validate_test.go b/base/v0_1/validate_test.go new file mode 100644 index 000000000..650fb7027 --- /dev/null +++ b/base/v0_1/validate_test.go @@ -0,0 +1,79 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_1 + +import ( + "reflect" + "testing" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +// TestValidateFileContents tests that multiple sources (i.e. urls and inline) are not allowed but zero or one sources are +func TestValidateFileContents(t *testing.T) { + tests := []struct { + in FileContents + out error + }{ + {}, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + FileContents{ + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + nil, + }, + { + FileContents{ + Inline: util.StrToPtr("hello"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + nil, + }, + { + FileContents{ + Source: util.StrToPtr("data:,hello"), + Inline: util.StrToPtr("hello"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + ErrInlineAndSource, + }, + } + + for i, test := range tests { + actual := test.in.Validate(path.New("yaml")) + expected := report.Report{} + // hardcode inline for now since that's the only place errors occur. Move into the + // test struct once there's more than one place + expected.AddOnError(path.New("yaml", "inline"), test.out) + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("#%d: expected %+v got %+v", i, expected, actual) + } + } +} diff --git a/go.mod b/go.mod index 527a71ead..61ac496a4 100644 --- a/go.mod +++ b/go.mod @@ -8,5 +8,6 @@ require ( github.com/coreos/ignition/v2 v2.0.0 github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a github.com/davecgh/go-spew v1.1.1 // indirect + github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae ) From 87e4e6b670fb66ac3f10517de3fcb4ec5b11eeaf Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 8 Jul 2019 16:39:59 -0700 Subject: [PATCH 020/877] build: add build script --- .gitignore | 1 + build | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 .gitignore create mode 100755 build diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..ba077a403 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin diff --git a/build b/build new file mode 100755 index 000000000..323ddc4e9 --- /dev/null +++ b/build @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu + +export GO111MODULE=on +export GOFLAGS=-mod=readonly +export CGO_ENABLED=1 + +NAME=fcct + +eval $(go env) +if [ -z ${BIN_PATH+a} ]; then + BIN_PATH=${PWD}/bin/${GOARCH} +fi + +echo "Building $NAME..." +go build -buildmode=pie -o ${BIN_PATH}/${NAME} internal/main.go From c75fce2811c5b275597aa542baf66855f2e32e0e Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 9 Jul 2019 13:19:06 -0700 Subject: [PATCH 021/877] build: turn off CGO We don't use CGO, disable it. --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 323ddc4e9..a46fc9d70 100755 --- a/build +++ b/build @@ -4,7 +4,7 @@ set -eu export GO111MODULE=on export GOFLAGS=-mod=readonly -export CGO_ENABLED=1 +export CGO_ENABLED=0 NAME=fcct From b375553f7beb9fa945cf57b3116bfea68257fa26 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 9 Jul 2019 13:20:01 -0700 Subject: [PATCH 022/877] build_release: add script Add script to build releases. --- build_releases | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 build_releases diff --git a/build_releases b/build_releases new file mode 100755 index 000000000..e7f43831a --- /dev/null +++ b/build_releases @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export GO111MODULE=on +export GOFLAGS=-mod=readonly +export CGO_ENABLED=0 + +eval $(go env) +if [ -z ${BIN_PATH+a} ]; then + export BIN_PATH=${PWD}/bin/releases/ +fi + +build_release() { + export NAME="fcct-${1}" + echo "building ${NAME}" + go build -o ${BIN_PATH}/${NAME} internal/main.go +} + +export GOOS=linux +export GOARCH=amd64 +build_release x86_64-unknown-linux-gnu + +export GOOS=darwin +export GOARCH=amd64 +build_release x86_64-apple-darwin + +export GOOS=windows +export GOARCH=amd64 +build_release x86_64-pc-windows-gnu.exe + +export GOOS=linux +export GOARCH=arm64 +build_release aarch64-unknown-linux-gnu From 559385a46582a15ea73ca8f7d6f39b7805c13a41 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 9 Jul 2019 13:21:37 -0700 Subject: [PATCH 023/877] tag_release: add script Add script to tag releases. Copied over from Ignition. --- tag_release.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 tag_release.sh diff --git a/tag_release.sh b/tag_release.sh new file mode 100755 index 000000000..7feea617b --- /dev/null +++ b/tag_release.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +[ $# == 2 ] || { echo "usage: $0 " && exit 1; } + +VER=$1 +COMMIT=$2 + +[[ "${VER}" =~ ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-.+)?$ ]] || { + echo "malformed version: \"${VER}\"" + exit 2 +} + +[[ "${COMMIT}" =~ ^[[:xdigit:]]+$ ]] || { + echo "malformed commit id: \"${COMMIT}\"" + exit 3 +} + +source ./build + +git tag --sign --message "Fedora CoreOS Config Transpiler ${VER}" "${VER}" "${COMMIT}" +git verify-tag --verbose "${VER}" From efd10b4da5069f4a8e29022d84a0535ee200bf77 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 9 Jul 2019 13:36:34 -0700 Subject: [PATCH 024/877] releases: add checklist and doc Add an issue template for creating releases. Add a development doc with a link to create new releases. --- .github/ISSUE_TEMPLATE/release-checklist.md | 16 ++++++++++++++++ docs/development.md | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/release-checklist.md create mode 100644 docs/development.md diff --git a/.github/ISSUE_TEMPLATE/release-checklist.md b/.github/ISSUE_TEMPLATE/release-checklist.md new file mode 100644 index 000000000..1e0953b26 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release-checklist.md @@ -0,0 +1,16 @@ +Release checklist: + - [ ] Write release notes in NEWS. Get them reviewed and merged + - [ ] If doing a branched release, also include a PR to merge the NEWS changes into master + - [ ] Ensure your local copy is up to date with master and your working directory is clean + - [ ] Ensure you can sign commits and any yubikeys/smartcards are plugged in + - [ ] Run `./tag_release.sh ` + - [ ] Push that tag to Github + - [ ] Run `./build_releases` + - [ ] Sign the release artifacts by running +``` +gpg --local-user 0xCDDE268EBB729EC7! --detach-sign --armor +``` +for each release artifact. Do not try to sign all of them at once by globbing. If you do, gpg will sign the combination of all the release artifacts instead of each one individually. + + - [ ] Create a draft release on Github and upload all the release artifacts and their signatures. Copy and paste the release notes from NEWS here as well. + - [ ] Publish the release diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 000000000..efc3d51b3 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,5 @@ +## Developing FCCT + +### Creating a release: + +Create a [release checklist](https://github.com/ajeddeloh/fcct/issues/new?template=release-checklist.md) and follow those steps. From 418bebaf8b68cc5a1b0ceab309855fd2ed054381 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 8 Jul 2019 14:25:28 -0700 Subject: [PATCH 025/877] docs: add doc describing schema This is just the Ignition doc with irrelevant bits removed and version and variant added. --- docs/configuration-v1_0.md | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 docs/configuration-v1_0.md diff --git a/docs/configuration-v1_0.md b/docs/configuration-v1_0.md new file mode 100644 index 000000000..471735379 --- /dev/null +++ b/docs/configuration-v1_0.md @@ -0,0 +1,125 @@ +# Configuration Specification v1.0.0 # + +The Fedora CoreOS configuration is a YAML document conforming to the following specification, with **_italicized_** entries being optional: + +* **variant** (string): must be `fcos`. +* **version** (string): the semantic version of the spec for this document. This document is for version `1.0.0` and generates Ignition configs with version `3.0.0`. +* **ignition** (object): metadata about the configuration itself. + * **_config_** (objects): options related to the configuration. + * **_merge_** (list of objects): a list of the configs to be merged to the current config. + * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the config. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_replace_** (object): the config that will replace the current. + * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the config. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_timeouts_** (object): options relating to `http` timeouts when fetching files over `http` or `https`. + * **_httpResponseHeaders_** (integer) the time to wait (in seconds) for the server's response headers (but not the body) after making a request. 0 indicates no timeout. Default is 10 seconds. + * **_httpTotal_** (integer) the time limit (in seconds) for the operation (connection, request, and response), including retries. 0 indicates no timeout. Default is 0. + * **_security_** (object): options relating to network security. + * **_tls_** (object): options relating to TLS when fetching resources over `https`. + * **_certificateAuthorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`. + * **source** (string): the URL of the certificate (in PEM format). Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the certificate. + * **_hash_** (string): the hash of the certificate, in the form `-` where type is sha512. +* **_storage_** (object): describes the desired state of the system's storage devices. + * **_disks_** (list of objects): the list of disks to be configured and their options. Every entry must have a unique `device`. + * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. + * **_wipeTable_** (boolean): whether or not the partition tables shall be wiped. When true, the partition tables are erased before any further manipulation. Otherwise, the existing entries are left intact. + * **_partitions_** (list of objects): the list of partitions and their configuration for this particular disk. Every partition must have a unique `number`, or if 0 is specified, a unique `label`. + * **_label_** (string): the PARTLABEL for the partition. + * **_number_** (integer): the partition number, which dictates it's position in the partition table (one-indexed). If zero, use the next available partition slot. + * **_sizeMiB_** (integer): the size of the partition (in mebibytes). If zero, the partition will be made as large as possible. + * **_startMiB_** (integer): the start of the partition (in mebibytes). If zero, the partition will be positioned at the start of the largest block available. + * **_typeGuid_** (string): the GPT [partition type GUID][part-types]. If omitted, the default will be 0FC63DAF-8483-4772-8E79-3D69D8477DE4 (Linux filesystem data). + * **_guid_** (string): the GPT unique partition GUID. + * **_wipePartitionEntry_** (boolean) if true, Ignition will clobber an existing partition if it does not match the config. If false (default), Ignition will fail instead. + * **_shouldExist_** (boolean) whether or not the partition with the specified `number` should exist. If omitted, it defaults to true. If false Ignition will either delete the specified partition or fail, depending on `wipePartitionEntry`. If false `number` must be specified and non-zero and `label`, `start`, `size`, `guid`, and `typeGuid` must all be omitted. + * **_raid_** (list of objects): the list of RAID arrays to be configured. Every RAID array must have a unique `name`. + * **name** (string): the name to use for the resulting md device. + * **level** (string): the redundancy level of the array (e.g. linear, raid1, raid5, etc.). + * **devices** (list of strings): the list of devices (referenced by their absolute path) in the array. + * **_spares_** (integer): the number of spares (if applicable) in the array. + * **_options_** (list of strings): any additional options to be passed to mdadm. + * **_filesystems_** (list of objects): the list of filesystems to be configured. `path`, `device`, and `format` all need to be specified. Every filesystem must have a unique `device`. + * **path** (string): the mount-point of the filesystem while Ignition is running relative to where the root filesystem will be mounted. This is not necessarily the same as where it should be mounted in the real root, but it is encouraged to make it the same. + * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. + * **format** (string): the filesystem format (ext4, btrfs, xfs, vfat, or swap). + * **_wipeFilesystem_** (boolean): whether or not to wipe the device before filesystem creation, see [the documentation on filesystems](operator-notes.md#filesystem-reuse-semantics) for more information. + * **_label_** (string): the label of the filesystem. + * **_uuid_** (string): the uuid of the filesystem. + * **_options_** (list of strings): any additional options to be passed to the format-specific mkfs utility. + * **_files_** (list of objects): the list of files to be written. Every file, directory and link must have a unique `path`. + * **path** (string): the absolute path to the file. + * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `source` must be specified if `overwrite` is true. Defaults to false. + * **_contents_** (object): options related to the contents of the file. + * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. + * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created. + * **_verification_** (object): options related to the verification of the file contents. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents` + * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. + * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the appended contents. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_mode_** (integer): the file's permission mode. If not specified, the permission mode for files defaults to 0644 or the existing file's permissions if `overwrite` is false, `source` is unspecified, and a file already exists at the path. + * **_user_** (object): specifies the file's owner. + * **_id_** (integer): the user ID of the owner. + * **_name_** (string): the user name of the owner. + * **_group_** (object): specifies the group of the owner. + * **_id_** (integer): the group ID of the owner. + * **_name_** (string): the group name of the owner. + * **_directories_** (list of objects): the list of directories to be created. Every file, directory, and link must have a unique `path`. + * **path** (string): the absolute path to the directory. + * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If false and a directory already exists at the path, Ignition will only set its permissions. If false and a non-directory exists at that path, Ignition will fail. Defaults to false. + * **_mode_** (integer): the directory's permission mode. If not specified, the permission mode for directories defaults to 0755 or the mode of an existing directory if `overwrite` is false and a directory already exists at the path. + * **_user_** (object): specifies the directory's owner. + * **_id_** (integer): the user ID of the owner. + * **_name_** (string): the user name of the owner. + * **_group_** (object): specifies the group of the owner. + * **_id_** (integer): the group ID of the owner. + * **_name_** (string): the group name of the owner. + * **_links_** (list of objects): the list of links to be created. Every file, directory, and link must have a unique `path`. + * **path** (string): the absolute path to the link + * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If overwrite is false and a matching link exists at the path, Ignition will only set the owner and group. Defaults to false. + * **_user_** (object): specifies the symbolic link's owner. + * **_id_** (integer): the user ID of the owner. + * **_name_** (string): the user name of the owner. + * **_group_** (object): specifies the group of the owner. + * **_id_** (integer): the group ID of the owner. + * **_name_** (string): the group name of the owner. + * **target** (string): the target path of the link + * **_hard_** (boolean): a symbolic link is created if this is false, a hard one if this is true. +* **_systemd_** (object): describes the desired state of the systemd units. + * **_units_** (list of objects): the list of systemd units. + * **name** (string): the name of the unit. This must be suffixed with a valid unit type (e.g. "thing.service"). Every unit must have a unique `name`. + * **_enabled_** (boolean): whether or not the service shall be enabled. When true, the service is enabled. When false, the service is disabled. When omitted, the service is unmodified. In order for this to have any effect, the unit must have an install section. + * **_mask_** (boolean): whether or not the service shall be masked. When true, the service is masked by symlinking it to `/dev/null`. + * **_contents_** (string): the contents of the unit. + * **_dropins_** (list of objects): the list of drop-ins for the unit. Every drop-in must have a unique `name`. + * **name** (string): the name of the drop-in. This must be suffixed with ".conf". + * **_contents_** (string): the contents of the drop-in. +* **_passwd_** (object): describes the desired additions to the passwd database. + * **_users_** (list of objects): the list of accounts that shall exist. All users must have a unique `name`. + * **name** (string): the username for the account. + * **_passwordHash_** (string): the encrypted password for the account. + * **_sshAuthorizedKeys_** (list of strings): a list of SSH keys to be added as an SSH key fragment at `.ssh/authorized_keys.d/ignition` in the user's home directory. All SSH keys must be unique. + * **_uid_** (integer): the user ID of the account. + * **_gecos_** (string): the GECOS field of the account. + * **_homeDir_** (string): the home directory of the account. + * **_noCreateHome_** (boolean): whether or not to create the user's home directory. This only has an effect if the account doesn't exist yet. + * **_primaryGroup_** (string): the name of the primary group of the account. + * **_groups_** (list of strings): the list of supplementary groups of the account. + * **_noUserGroup_** (boolean): whether or not to create a group with the same name as the user. This only has an effect if the account doesn't exist yet. + * **_noLogInit_** (boolean): whether or not to add the user to the lastlog and faillog databases. This only has an effect if the account doesn't exist yet. + * **_shell_** (string): the login shell of the new account. + * **_system_** (bool): whether or not this account should be a system account. This only has an effect if the account doesn't exist yet. + * **_groups_** (list of objects): the list of groups to be added. All groups must have a unique `name`. + * **name** (string): the name of the group. + * **_gid_** (integer): the group ID of the new group. + * **_passwordHash_** (string): the encrypted password of the new group. + * **_system_** (bool): whether or not the group should be a system group. This only has an effect if the group doesn't exist yet. + +[part-types]: http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs +[rfc2397]: https://tools.ietf.org/html/rfc2397 From 3edba662ad40ccf90ad359c01417d876f29e6f72 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 8 Jul 2019 14:41:33 -0700 Subject: [PATCH 026/877] docs: add getting started guide This is mostly copied from the CL getting started guide. --- docs/getting-started.md | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/getting-started.md diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 000000000..32ea1b8d9 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,50 @@ +# Getting started + +`fcct`, the Fedora CoreOS Config Transpiler, is a tool that consumes a Fedora CoreOS Config and produces an Ignition config, which is a JSON document that can be given to a Fedora CoreOS machine when it first boots. Using this config, a machine can be told to create users, create filesystems, set up the network, install systemd units, and more. + +Fedora CoreOS Configs are YAML files conforming to `fcct`'s schema. For more information on the schema, take a look at [doc/configuration-v1_0.md][spec]. + +### Getting FCCT + +Download the latest version of `fcct` and the detached signature from the [releases page](https://github.com/ajeddeloh/fcct/releases). Verify it with gpg: + +``` +gpg --verify +``` + +New releases of `fcct` are backwards compatible with old releases unless otherwise noted. + +### Writing and using Fedora CoreOS Configs + +As a simple example, let's use `fcct` to set the authorized ssh key for the `core` user on a Fedora CoreOS machine. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +passwd: + users: + - name: core + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc... +``` + +In this above file, you'll want to set the `ssh-rsa AAAAB3NzaC1yc...` line to be your ssh public key (which is probably the contents of `~/.ssh/id_rsa.pub`, if you're on Linux). + +If we take this file and give it to `fcct`: + +``` +$ ./bin/amd64/fcct --input example.yaml + +{"ignition":{"config":{"replace":{"source":null,"verification":{}}},"security":{"tls":{}},"timeouts":{},"version":"3.0.0"},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa ssh-rsa AAAAB3NzaC1yc..."]}]},"storage":{},"systemd":{}} +``` + +We can see that it produces a JSON file. This file isn't intended to be human-friendly, and will definitely be a pain to read/edit (especially if you have multi-line things like systemd units). Luckily, you shouldn't have to care about this file! Just provide it to a booting Fedora CoreOS machine and [Ignition][ignition], the utility inside of Fedora CoreOS that receives this file, will know what to do with it. + +The method by which this file is provided to a Fedora CoreOS machine depends on the environment in which the machine is running. For instructions on a given provider, head over to the [list of supported platforms for Ignition][supported-platforms]. + +To see some examples for what else `fcct` can do, head over to the [examples][examples]. + +[spec]: configuration-v1_0.md +[ignition]: https://github.com/coreos/ignition +[supported-platforms]: https://github.com/coreos/ignition/blob/master/doc/supported-platforms.md +[examples]: examples.md From 0413321bb05ea40b8358581eee7708d1c8aa371f Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 8 Jul 2019 15:58:06 -0700 Subject: [PATCH 027/877] docs: add examples Add some example fcc files --- docs/examples.md | 147 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 docs/examples.md diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 000000000..a336722c6 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,147 @@ +# Examples + +Here you can find a bunch of simple examples for using `fcct`, with some explanations about what they do. The examples here are in no way comprehensive, for a full list of all the options present in `fcct` check out the [configuration specification][spec]. + +## Users and groups + +This example modifies the existing `core` user and sets its ssh key. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +passwd: + users: + - name: core + ssh_authorized_keys: + - key1 +``` + +This example creates one user, `user1` and sets up one ssh public key for the user. The user is also given the home directory `/home/user1`, but it's not created, the user is added to the `wheel` and `plugdev` groups, and the user's shell is set to `/bin/bash`. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +passwd: + users: + - name: user1 + ssh_authorized_keys: + - key1 + home_dir: /home/user1 + no_create_home: true + groups: + - wheel + - plugdev + shell: /bin/bash +``` + +## Storage and files + +### Files + +This example creates a file at `/opt/file` with the contents `Hello, world!`, permissions 0644 (so readable and writable by the owner, and only readable by everyone else), and the file is owned by user uid 500 and gid 501. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +storage: + files: + - path: /opt/file + contents: + inline: Hello, world! + mode: 0644 + user: + id: 500 + group: + id: 501 +``` + +This example fetches a gzip-compressed file from `http://example.com/file2`, makes sure that it matches the provided sha512 hash, and writes it to `/opt/file2`. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +storage: + files: + - path: /opt/file2 + contents: + source: http://example.com/file2 + compression: gzip + verification: + hash: sha512-4ee6a9d20cc0e6c7ee187daffa6822bdef7f4cebe109eff44b235f97e45dc3d7a5bb932efc841192e46618f48a6f4f5bc0d15fd74b1038abf46bf4b4fd409f2e + mode: 0644 +``` + +## systemd units + +This example adds a drop-in for the `serial-getty@ttyS0` unit, turning on autologin on `ttyS0` by overriding the `ExecStart=` defined in the default unit. More information on systemd dropins can be found in [the systemd docs][dropins]. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +systemd: + units: + - name: serial-getty@ttyS0.service + dropins: + - name: autologin.conf + contents: | + [Service] + TTYVTDisallocate=no + ExecStart= + ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM +``` + +This example creates a new systemd unit called hello.service, enables it so it will run on boot, and defines the contents to simply echo `"Hello, World!"`. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +systemd: + units: + - name: hello.service + enabled: true + contents: | + [Unit] + Description=A hello world unit! + [Service] + Type=oneshot + ExecStart=/usr/bin/echo "Hello, World!" + [Install] + WantedBy=multi-user.target +``` + +### Filesystems and Partitions + +This example creates a single partition spanning all of the sdb device then creates a btrfs filesystem on it to use as /var. Finally it creates the mount unit for systemd so it gets mounted on boot. + +```yaml fedora-coreos-config +variant: fcos +version: 1.0.0 +storage: + disks: + - device: /dev/sdb + wipe_table: true + partitions: + - number: 1 + label: var + filesystems: + - path: /var + device: /dev/disk/by-partlabel/var + format: btrfs + wipe_filesystem: true + label: var +systemd: + units: + - name: var.mount + enabled: true + contents: | + [Unit] + Before=local-fs.target + [Mount] + Where=/var + What=/dev/disk/by-partlabel/var + [Install] + WantedBy=local-fs.target +``` + +[spec]: configuration.md +[dropins]: https://www.freedesktop.org/software/systemd/man/systemd.unit.html#Description From dd84656d17b5edc9d018f89efdfadd656a8f403c Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 8 Jul 2019 16:10:07 -0700 Subject: [PATCH 028/877] README: update readme to link to getting started Add an explainer blurb that points to better docs, convert to markdown. --- README => README.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) rename README => README.md (68%) diff --git a/README b/README.md similarity index 68% rename from README rename to README.md index 684ff5be5..72e89423a 100644 --- a/README +++ b/README.md @@ -1,9 +1,14 @@ -Fedora CoreOS Config Transpiler +# Fedora CoreOS Config Transpiler -Still a work in progress. +The Fedora CoreOS Config Transpiler (FCCT) translates human readable Fedora CoreOS Configs (FCCs) +into machine readable [Ignition](https://github.com/coreos/ignition) Configs. See the [getting +started](docs/getting-started.md) guide for how to use FCCT and the [configuration spec](docs/configuration-v1_0.md) +for everything FCCs support. + +### Project Layout Each config spec is composed of a base and distro component. The base components -roughly mirror the Ignition spec and are distro agnostic. The distro compoents +roughly mirror the Ignition spec and are distro agnostic. The distro components contain sugar for common configuration of the host (e.g. etcd) and are not distro-independent. @@ -12,30 +17,30 @@ version getting it's own package. These versions are not exposed to the user. Each fcos config version has it's own version which is independent of the versions of the base and distro components that compose it. However a major -or minor bump of either component necesitates a corresponding bump in the fcos +or minor bump of either component necessitates a corresponding bump in the fcos config version. -internal/ +`internal/` main, non-exported code -base/ +`base/` Contains distro-agnostic code. Each package here targets only one Ignition - spec verions. + spec versions. -distro/ +`distro/` Contains distro-specific code. Each package here can target multiple Ignition versions if it makes sense. -config/ +`config/` Contains the top level Translate() function that determines which version to parse and emit. -config/common/ +`config/common/` Contains the common bits and functions for all spec versions. This means the (un)marshaling helpers and the version+variant struct to be included in every user facing spec -config/vX_Y/ +`config/vX_Y/` Contains user facing definitions of the spec. Each is composed by combining a base and distro package with the common version+variant. Each of the defines their own translate function to be registered in the config/ package. From e82da78c9eea59274888ddbf1745d876df54c403 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 10 Jul 2019 13:17:51 -0700 Subject: [PATCH 029/877] *: move to coreos org Fix all the import paths and docs to use github.com/coreos/fcct instead of github.com/ajeddeloh/fcct. --- config/config.go | 4 ++-- config/v1_0/fcos.go | 6 +++--- docs/development.md | 2 +- docs/getting-started.md | 2 +- go.mod | 2 +- internal/main.go | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index 7293c4440..fcf18be8c 100644 --- a/config/config.go +++ b/config/config.go @@ -18,8 +18,8 @@ import ( "errors" "fmt" - "github.com/ajeddeloh/fcct/config/common" - "github.com/ajeddeloh/fcct/config/v1_0" + "github.com/coreos/fcct/config/common" + "github.com/coreos/fcct/config/v1_0" "github.com/coreos/go-semver/semver" "gopkg.in/yaml.v3" diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index edfc49832..623e58bf6 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -19,9 +19,9 @@ import ( "fmt" "reflect" - base_0_1 "github.com/ajeddeloh/fcct/base/v0_1" - "github.com/ajeddeloh/fcct/config/common" - fcos_0_1 "github.com/ajeddeloh/fcct/distro/fcos/v0_1" + base_0_1 "github.com/coreos/fcct/base/v0_1" + "github.com/coreos/fcct/config/common" + fcos_0_1 "github.com/coreos/fcct/distro/fcos/v0_1" "github.com/coreos/ignition/v2/config/v3_0" "github.com/coreos/ignition/v2/config/v3_0/types" diff --git a/docs/development.md b/docs/development.md index efc3d51b3..f6d74d2fc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,4 +2,4 @@ ### Creating a release: -Create a [release checklist](https://github.com/ajeddeloh/fcct/issues/new?template=release-checklist.md) and follow those steps. +Create a [release checklist](https://github.com/coreos/fcct/issues/new?template=release-checklist.md) and follow those steps. diff --git a/docs/getting-started.md b/docs/getting-started.md index 32ea1b8d9..c7bf681b3 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,7 +6,7 @@ Fedora CoreOS Configs are YAML files conforming to `fcct`'s schema. For more inf ### Getting FCCT -Download the latest version of `fcct` and the detached signature from the [releases page](https://github.com/ajeddeloh/fcct/releases). Verify it with gpg: +Download the latest version of `fcct` and the detached signature from the [releases page](https://github.com/coreos/fcct/releases). Verify it with gpg: ``` gpg --verify diff --git a/go.mod b/go.mod index 61ac496a4..0db4347ef 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/ajeddeloh/fcct +module github.com/coreos/fcct go 1.12 diff --git a/internal/main.go b/internal/main.go index f9481b562..e522bca35 100644 --- a/internal/main.go +++ b/internal/main.go @@ -20,8 +20,8 @@ import ( "io/ioutil" "os" - "github.com/ajeddeloh/fcct/config" - "github.com/ajeddeloh/fcct/config/common" + "github.com/coreos/fcct/config" + "github.com/coreos/fcct/config/common" ) func fail(format string, args ...interface{}) { From f0ddd1f979ca9148483a66de17d74dcb6131ad03 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 10 Jul 2019 15:24:58 -0700 Subject: [PATCH 030/877] NEWS: Add NEWS file, news for v0.1.0 --- NEWS | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 NEWS diff --git a/NEWS b/NEWS new file mode 100644 index 000000000..20843ebd3 --- /dev/null +++ b/NEWS @@ -0,0 +1,5 @@ +2019-07-10 FCCT 0.1.0 + +Initial Release of FCCT. While the golang API is not stable, the Fedora CoreOS +Configuration language is. Configs written with version 1.0.0 will continue to +work with future releases of FCCT. From 9b0487ccac673e88a9d14a4f2eb9d0a672b7d8d4 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 10 Jul 2019 15:33:35 -0700 Subject: [PATCH 031/877] build: don't use -buildmode=pie This isn't needed and breaks without cgo enabled. --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index a46fc9d70..86b6f62b8 100755 --- a/build +++ b/build @@ -14,4 +14,4 @@ if [ -z ${BIN_PATH+a} ]; then fi echo "Building $NAME..." -go build -buildmode=pie -o ${BIN_PATH}/${NAME} internal/main.go +go build -o ${BIN_PATH}/${NAME} internal/main.go From 69ad9e1c947da9ad1578e81d2f09005c8be43432 Mon Sep 17 00:00:00 2001 From: Stephen Greene Date: Wed, 17 Jul 2019 11:56:38 -0700 Subject: [PATCH 032/877] Fix gpg verification command in docs --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index c7bf681b3..ef89b171e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -9,7 +9,7 @@ Fedora CoreOS Configs are YAML files conforming to `fcct`'s schema. For more inf Download the latest version of `fcct` and the detached signature from the [releases page](https://github.com/coreos/fcct/releases). Verify it with gpg: ``` -gpg --verify +gpg --verify ``` New releases of `fcct` are backwards compatible with old releases unless otherwise noted. From a60995b54eabdcf72c4e8d7af143c35443820af6 Mon Sep 17 00:00:00 2001 From: Stephen Greene Date: Wed, 17 Jul 2019 14:59:35 -0700 Subject: [PATCH 033/877] main: Fix output file handling --- internal/main.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/main.go b/internal/main.go index e522bca35..45da51b2b 100644 --- a/internal/main.go +++ b/internal/main.go @@ -48,31 +48,31 @@ func main() { var err error infile, err = os.Open(input) if err != nil { - fail("failed to open %s: %v", input, err) + fail("failed to open %s: %v\n", input, err) } defer infile.Close() } dataIn, err := ioutil.ReadAll(infile) if err != nil { - fail("failed to read %s: %v", infile.Name(), err) + fail("failed to read %s: %v\n", infile.Name(), err) } dataOut, err := config.Translate(dataIn, options) if err != nil { - fail("Error translating config: %v", err) + fail("Error translating config: %v\n", err) } if output != "" { var err error - outfile, err = os.Open(output) + outfile, err = os.OpenFile(output, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { - fail("failed to open %s: %v", output, err) + fail("failed to open %s: %v\n", output, err) } defer outfile.Close() } if _, err := outfile.Write(dataOut); err != nil { - fail("Failed to write config to %s: %v", outfile.Name(), err) + fail("Failed to write config to %s: %v\n", outfile.Name(), err) } } From 32c6019800967fb60424a7f5ee9d2a91412a658f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 18 Jul 2019 09:33:06 -0400 Subject: [PATCH 034/877] docs/examples: fix link to config spec Closes: #17 --- docs/examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples.md b/docs/examples.md index a336722c6..955bb791c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -143,5 +143,5 @@ systemd: WantedBy=local-fs.target ``` -[spec]: configuration.md +[spec]: configuration-v1_0.md [dropins]: https://www.freedesktop.org/software/systemd/man/systemd.unit.html#Description From 452251dc74faa9fc13e6625b1442d990e47ac5ba Mon Sep 17 00:00:00 2001 From: Stephen Greene Date: Fri, 19 Jul 2019 10:21:41 -0700 Subject: [PATCH 035/877] Docs: Update gpg verify instructions Closes #13 --- docs/getting-started.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index ef89b171e..c38105de4 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,6 +11,7 @@ Download the latest version of `fcct` and the detached signature from the [relea ``` gpg --verify ``` +You may need to download the [CoreOS Application Signing Key](http://coreos.com/security/app-signing-key/) and import it with `gpg --import ` if you have not already done so. New releases of `fcct` are backwards compatible with old releases unless otherwise noted. From 6e9055423fad3eb8a32866124c7046ea21e23b49 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 19 Jul 2019 14:58:29 -0700 Subject: [PATCH 036/877] docs/configuration: fix camelCase -> snake_case --- docs/configuration-v1_0.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/configuration-v1_0.md b/docs/configuration-v1_0.md index 471735379..098f83ac5 100644 --- a/docs/configuration-v1_0.md +++ b/docs/configuration-v1_0.md @@ -15,27 +15,27 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **_verification_** (object): options related to the verification of the config. * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. * **_timeouts_** (object): options relating to `http` timeouts when fetching files over `http` or `https`. - * **_httpResponseHeaders_** (integer) the time to wait (in seconds) for the server's response headers (but not the body) after making a request. 0 indicates no timeout. Default is 10 seconds. - * **_httpTotal_** (integer) the time limit (in seconds) for the operation (connection, request, and response), including retries. 0 indicates no timeout. Default is 0. + * **_http_response_headers_** (integer) the time to wait (in seconds) for the server's response headers (but not the body) after making a request. 0 indicates no timeout. Default is 10 seconds. + * **_http_total_** (integer) the time limit (in seconds) for the operation (connection, request, and response), including retries. 0 indicates no timeout. Default is 0. * **_security_** (object): options relating to network security. * **_tls_** (object): options relating to TLS when fetching resources over `https`. - * **_certificateAuthorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`. + * **_certificate_authorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`. * **source** (string): the URL of the certificate (in PEM format). Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_verification_** (object): options related to the verification of the certificate. * **_hash_** (string): the hash of the certificate, in the form `-` where type is sha512. * **_storage_** (object): describes the desired state of the system's storage devices. * **_disks_** (list of objects): the list of disks to be configured and their options. Every entry must have a unique `device`. * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. - * **_wipeTable_** (boolean): whether or not the partition tables shall be wiped. When true, the partition tables are erased before any further manipulation. Otherwise, the existing entries are left intact. + * **_wipe_table_** (boolean): whether or not the partition tables shall be wiped. When true, the partition tables are erased before any further manipulation. Otherwise, the existing entries are left intact. * **_partitions_** (list of objects): the list of partitions and their configuration for this particular disk. Every partition must have a unique `number`, or if 0 is specified, a unique `label`. * **_label_** (string): the PARTLABEL for the partition. * **_number_** (integer): the partition number, which dictates it's position in the partition table (one-indexed). If zero, use the next available partition slot. - * **_sizeMiB_** (integer): the size of the partition (in mebibytes). If zero, the partition will be made as large as possible. - * **_startMiB_** (integer): the start of the partition (in mebibytes). If zero, the partition will be positioned at the start of the largest block available. - * **_typeGuid_** (string): the GPT [partition type GUID][part-types]. If omitted, the default will be 0FC63DAF-8483-4772-8E79-3D69D8477DE4 (Linux filesystem data). + * **_size_mib_** (integer): the size of the partition (in mebibytes). If zero, the partition will be made as large as possible. + * **_start_mib_** (integer): the start of the partition (in mebibytes). If zero, the partition will be positioned at the start of the largest block available. + * **_type_guid_** (string): the GPT [partition type GUID][part-types]. If omitted, the default will be 0FC63DAF-8483-4772-8E79-3D69D8477DE4 (Linux filesystem data). * **_guid_** (string): the GPT unique partition GUID. - * **_wipePartitionEntry_** (boolean) if true, Ignition will clobber an existing partition if it does not match the config. If false (default), Ignition will fail instead. - * **_shouldExist_** (boolean) whether or not the partition with the specified `number` should exist. If omitted, it defaults to true. If false Ignition will either delete the specified partition or fail, depending on `wipePartitionEntry`. If false `number` must be specified and non-zero and `label`, `start`, `size`, `guid`, and `typeGuid` must all be omitted. + * **_wipe_partition_entry_** (boolean) if true, Ignition will clobber an existing partition if it does not match the config. If false (default), Ignition will fail instead. + * **_should_exist_** (boolean) whether or not the partition with the specified `number` should exist. If omitted, it defaults to true. If false Ignition will either delete the specified partition or fail, depending on `wipePartitionEntry`. If false `number` must be specified and non-zero and `label`, `start`, `size`, `guid`, and `typeGuid` must all be omitted. * **_raid_** (list of objects): the list of RAID arrays to be configured. Every RAID array must have a unique `name`. * **name** (string): the name to use for the resulting md device. * **level** (string): the redundancy level of the array (e.g. linear, raid1, raid5, etc.). @@ -46,7 +46,7 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **path** (string): the mount-point of the filesystem while Ignition is running relative to where the root filesystem will be mounted. This is not necessarily the same as where it should be mounted in the real root, but it is encouraged to make it the same. * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. * **format** (string): the filesystem format (ext4, btrfs, xfs, vfat, or swap). - * **_wipeFilesystem_** (boolean): whether or not to wipe the device before filesystem creation, see [the documentation on filesystems](operator-notes.md#filesystem-reuse-semantics) for more information. + * **_wipe_filesystem_** (boolean): whether or not to wipe the device before filesystem creation, see [the documentation on filesystems](operator-notes.md#filesystem-reuse-semantics) for more information. * **_label_** (string): the label of the filesystem. * **_uuid_** (string): the uuid of the filesystem. * **_options_** (list of strings): any additional options to be passed to the format-specific mkfs utility. @@ -103,22 +103,22 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **_passwd_** (object): describes the desired additions to the passwd database. * **_users_** (list of objects): the list of accounts that shall exist. All users must have a unique `name`. * **name** (string): the username for the account. - * **_passwordHash_** (string): the encrypted password for the account. - * **_sshAuthorizedKeys_** (list of strings): a list of SSH keys to be added as an SSH key fragment at `.ssh/authorized_keys.d/ignition` in the user's home directory. All SSH keys must be unique. + * **_password_hash_** (string): the encrypted password for the account. + * **_ssh_authorized_keys_** (list of strings): a list of SSH keys to be added as an SSH key fragment at `.ssh/authorized_keys.d/ignition` in the user's home directory. All SSH keys must be unique. * **_uid_** (integer): the user ID of the account. * **_gecos_** (string): the GECOS field of the account. - * **_homeDir_** (string): the home directory of the account. - * **_noCreateHome_** (boolean): whether or not to create the user's home directory. This only has an effect if the account doesn't exist yet. - * **_primaryGroup_** (string): the name of the primary group of the account. + * **_home_dir_** (string): the home directory of the account. + * **_no_create_home_** (boolean): whether or not to create the user's home directory. This only has an effect if the account doesn't exist yet. + * **_primary_group_** (string): the name of the primary group of the account. * **_groups_** (list of strings): the list of supplementary groups of the account. - * **_noUserGroup_** (boolean): whether or not to create a group with the same name as the user. This only has an effect if the account doesn't exist yet. - * **_noLogInit_** (boolean): whether or not to add the user to the lastlog and faillog databases. This only has an effect if the account doesn't exist yet. + * **_no_user_group_** (boolean): whether or not to create a group with the same name as the user. This only has an effect if the account doesn't exist yet. + * **_no_log_init_** (boolean): whether or not to add the user to the lastlog and faillog databases. This only has an effect if the account doesn't exist yet. * **_shell_** (string): the login shell of the new account. * **_system_** (bool): whether or not this account should be a system account. This only has an effect if the account doesn't exist yet. * **_groups_** (list of objects): the list of groups to be added. All groups must have a unique `name`. * **name** (string): the name of the group. * **_gid_** (integer): the group ID of the new group. - * **_passwordHash_** (string): the encrypted password of the new group. + * **_password_hash_** (string): the encrypted password of the new group. * **_system_** (bool): whether or not the group should be a system group. This only has an effect if the group doesn't exist yet. [part-types]: http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs From 35e4b83666defbf0abc9e388b491b697e184baf9 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 19 Jul 2019 15:00:23 -0700 Subject: [PATCH 037/877] docs: add explanation of variant --- docs/configuration-v1_0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration-v1_0.md b/docs/configuration-v1_0.md index 098f83ac5..83eae0458 100644 --- a/docs/configuration-v1_0.md +++ b/docs/configuration-v1_0.md @@ -2,7 +2,7 @@ The Fedora CoreOS configuration is a YAML document conforming to the following specification, with **_italicized_** entries being optional: -* **variant** (string): must be `fcos`. +* **variant** (string): used to differentiate configs for different operating systems. Must be `fcos` for FCCT. * **version** (string): the semantic version of the spec for this document. This document is for version `1.0.0` and generates Ignition configs with version `3.0.0`. * **ignition** (object): metadata about the configuration itself. * **_config_** (objects): options related to the configuration. From 1236ebbf950b70b8e37b77b19199a57674be05e2 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 22 Jul 2019 13:20:11 -0700 Subject: [PATCH 038/877] main: add --version flag --- build | 4 +++- build_releases | 4 +++- internal/main.go | 12 ++++++++++-- internal/version/version.go | 24 ++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 internal/version/version.go diff --git a/build b/build index 86b6f62b8..f7eb293b2 100755 --- a/build +++ b/build @@ -5,6 +5,8 @@ set -eu export GO111MODULE=on export GOFLAGS=-mod=readonly export CGO_ENABLED=0 +version=$(git describe --dirty --always) +LDFLAGS="-w -X github.com/coreos/fcct/internal/version.Raw=$version" NAME=fcct @@ -14,4 +16,4 @@ if [ -z ${BIN_PATH+a} ]; then fi echo "Building $NAME..." -go build -o ${BIN_PATH}/${NAME} internal/main.go +go build -o ${BIN_PATH}/${NAME} -ldflags "$LDFLAGS" internal/main.go diff --git a/build_releases b/build_releases index e7f43831a..e06dadae4 100755 --- a/build_releases +++ b/build_releases @@ -5,6 +5,8 @@ set -euo pipefail export GO111MODULE=on export GOFLAGS=-mod=readonly export CGO_ENABLED=0 +version=$(git describe --dirty --always) +LDFLAGS="-w -X github.com/coreos/fcct/internal/version.Raw=$version" eval $(go env) if [ -z ${BIN_PATH+a} ]; then @@ -14,7 +16,7 @@ fi build_release() { export NAME="fcct-${1}" echo "building ${NAME}" - go build -o ${BIN_PATH}/${NAME} internal/main.go + go build -o ${BIN_PATH}/${NAME} -ldflags "$LDFLAGS" internal/main.go } export GOOS=linux diff --git a/internal/main.go b/internal/main.go index 45da51b2b..16a9fed68 100644 --- a/internal/main.go +++ b/internal/main.go @@ -22,6 +22,7 @@ import ( "github.com/coreos/fcct/config" "github.com/coreos/fcct/config/common" + "github.com/coreos/fcct/internal/version" ) func fail(format string, args ...interface{}) { @@ -31,10 +32,12 @@ func fail(format string, args ...interface{}) { func main() { var ( - input string - output string + input string + output string + versionFlag bool ) options := common.TranslateOptions{} + flag.BoolVar(&versionFlag, "version", false, "print the version and exit") flag.BoolVar(&options.Strict, "strict", false, "fail on any warning") flag.BoolVar(&options.Pretty, "pretty", false, "output formatted json") flag.StringVar(&input, "input", "", "read from input file instead of stdin") @@ -42,6 +45,11 @@ func main() { flag.Parse() + if versionFlag { + fmt.Println(version.String) + os.Exit(0) + } + var infile *os.File = os.Stdin var outfile *os.File = os.Stdout if input != "" { diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 000000000..bd7d791be --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,24 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" +) + +var ( + Raw = "was not built properly" + String = fmt.Sprintf("Fedora CoreOS Config Transpiler %s", Raw) +) From 4c7945cfc2e378443082f37ec674f14f7abab058 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 19 Jul 2019 14:30:44 -0700 Subject: [PATCH 039/877] Dockerfile: add dockerfile for releases Add a dockerfile for building release containers. --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..643fd5404 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:latest AS builder +RUN mkdir /fcct +COPY . /fcct +WORKDIR /fcct +RUN ./build_releases + +FROM scratch +COPY --from=builder /fcct/bin/releases/fcct-x86_64-unknown-linux-gnu /usr/local/bin/fcct +ENTRYPOINT ["/usr/local/bin/fcct"] From 285e0b074a0a04b327908d6bbeb2a993354eb803 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 22 Jul 2019 13:57:21 -0700 Subject: [PATCH 040/877] test: add checking for examples in docs Extract and try all the examples in the docs as part of the test process to ensure we don't include invalid examples. --- test | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test b/test index b13eb9d02..c6ab06232 100755 --- a/test +++ b/test @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail export GO111MODULE=on @@ -8,5 +8,27 @@ res=$(gofmt -d .) echo "$res" test -z "$res" -echo "running tests" +echo "Running tests" go test ./... -cover + +echo "Checking docs" +shopt -s nullglob +mkdir tmpdocs +trap 'rm -r tmpdocs' EXIT + +for doc in docs/*md +do + echo "Checking $doc" + # split each doc into a bunch of tmpfiles then run ct on them + sed -n '/^```yaml fedora-coreos-config/,/^```/ p' < ${doc} \ + | csplit - '/```yaml fedora-coreos-config/' '{*}' -z --prefix tmpdocs/tmpconfig -q + + for i in tmpdocs/* + do + cat "$i" | tail -n +2 | head -n -1 \ + | go run internal/main.go --strict > /dev/null \ + || (cat -n "$i" && false) + done +done + +echo ok From 452fb922c3df13b3bbeaa417c698033d33f024f3 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 22 Jul 2019 14:42:02 -0700 Subject: [PATCH 041/877] travis: add travis.yaml --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..d791f25ed --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: go +go: + - "1.12.x" + +install: echo nop + +script: + - ./test From 28ae8a6efc79e84da7b3c3340d7cf41555870a00 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 25 Jul 2019 14:05:50 -0700 Subject: [PATCH 042/877] go.mod: bump ignition to 2.0.1 This includes a validation fix that file and directory paths. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0db4347ef..18632dda0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.12 require ( github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect - github.com/coreos/ignition/v2 v2.0.0 + github.com/coreos/ignition/v2 v2.0.1 github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a github.com/davecgh/go-spew v1.1.1 // indirect github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb diff --git a/go.sum b/go.sum index 89d68b24b..e1ab40f6b 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/ignition/v2 v2.0.0 h1:2nIU6rQIh2bCSF0PNJKuNNa23L13G031CoxbI4GFojc= -github.com/coreos/ignition/v2 v2.0.0/go.mod h1:EJP9Gk/u21BPwWa5nT6yYbm5mO0u8JQAAzFqaxgH7Fg= +github.com/coreos/ignition/v2 v2.0.1 h1:aKZARDpl6rDr83Ur/fUoQ3uuSM0VCLH5nTERkLczJFM= +github.com/coreos/ignition/v2 v2.0.1/go.mod h1:EJP9Gk/u21BPwWa5nT6yYbm5mO0u8JQAAzFqaxgH7Fg= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a h1:AdSIR3t2/3oreDLKnsCk8Lu5L3+OKlt9Yu7equBivmE= github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= From 953ea5cff0cf2644fd1572f6c9d8433abbccb91a Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 25 Jul 2019 14:51:24 -0700 Subject: [PATCH 043/877] news: add news for 0.2.0 --- NEWS | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/NEWS b/NEWS index 20843ebd3..4c5546786 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,21 @@ +2019-07-24 FCCT 0.2.0 + + Features: + + - Add --version flag + - Add Dockerfile and build containers automatically on quay.io + + Bug Fixes: + + - Fix validation of paths for files and directories + - Fix --output flag handling + + Misc Changes: + + - Add tests for the examples in the docs + - Add travis integration + + 2019-07-10 FCCT 0.1.0 Initial Release of FCCT. While the golang API is not stable, the Fedora CoreOS From 265d41e25b248391dc38921b84faf1604e3203a5 Mon Sep 17 00:00:00 2001 From: Stefan Bischof <33224746+stbischof@users.noreply.github.com> Date: Mon, 29 Jul 2019 17:19:41 +0200 Subject: [PATCH 044/877] [docs] passwords are hashed not encrypted --- docs/configuration-v1_0.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration-v1_0.md b/docs/configuration-v1_0.md index 83eae0458..6bcbce42c 100644 --- a/docs/configuration-v1_0.md +++ b/docs/configuration-v1_0.md @@ -103,7 +103,7 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **_passwd_** (object): describes the desired additions to the passwd database. * **_users_** (list of objects): the list of accounts that shall exist. All users must have a unique `name`. * **name** (string): the username for the account. - * **_password_hash_** (string): the encrypted password for the account. + * **_password_hash_** (string): the hashed password for the account. * **_ssh_authorized_keys_** (list of strings): a list of SSH keys to be added as an SSH key fragment at `.ssh/authorized_keys.d/ignition` in the user's home directory. All SSH keys must be unique. * **_uid_** (integer): the user ID of the account. * **_gecos_** (string): the GECOS field of the account. @@ -118,7 +118,7 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **_groups_** (list of objects): the list of groups to be added. All groups must have a unique `name`. * **name** (string): the name of the group. * **_gid_** (integer): the group ID of the new group. - * **_password_hash_** (string): the encrypted password of the new group. + * **_password_hash_** (string): the hashed password of the new group. * **_system_** (bool): whether or not the group should be a system group. This only has an effect if the group doesn't exist yet. [part-types]: http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs From 976d29bcd5a772a02dfc2d08a1920044f0cfb486 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 25 Jul 2019 16:09:38 -0700 Subject: [PATCH 045/877] docs/getting-started: add fcct container docs We publish fcct containers on quay. Document that and add instructions. --- docs/getting-started.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index c38105de4..fb95b4816 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,6 +6,10 @@ Fedora CoreOS Configs are YAML files conforming to `fcct`'s schema. For more inf ### Getting FCCT +`fcct` can be downloaded as a standalone binary or run as a container with docker or podman. + +#### Standalone binary + Download the latest version of `fcct` and the detached signature from the [releases page](https://github.com/coreos/fcct/releases). Verify it with gpg: ``` @@ -15,6 +19,21 @@ You may need to download the [CoreOS Application Signing Key](http://coreos.com/ New releases of `fcct` are backwards compatible with old releases unless otherwise noted. +#### Container + +This example uses podman, but docker can also be used. Substitute v0.2.0 with the desired version. Note that the `latest` tag corresponds with `master` and not with the latest release. + +```bash +# Pull the desired version +podman pull quay.io/coreos/fcct:v0.2.0 + +# Run fcct using standard in and standard out +podman run -it --rm quay.io/coreos/fcct:v0.2.0 -pretty -strict < your_config.fcc > transpiled_config.ign + +# Run fcct using files. +podman run --rm -v your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign +``` + ### Writing and using Fedora CoreOS Configs As a simple example, let's use `fcct` to set the authorized ssh key for the `core` user on a Fedora CoreOS machine. From 5b22d9000bf28a2f4804ad625f8d2cb705158f77 Mon Sep 17 00:00:00 2001 From: Robert Fairley Date: Tue, 20 Aug 2019 10:14:59 -0400 Subject: [PATCH 046/877] docs/getting-started: document absolute path for config bind mount Document using an absolute path to `your_config.fcc`. If an absolute path isn't supplied, `podman` interprets this as the name for a volume, and mounts the volume at `/config.fcc` in the container. With `/path/to`, it is clearer from the error message what the problem in the invoked command is if left unchanged. ``` $ podman run --rm -v your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign failed to read /config.fcc: read /config.fcc: is a directory $ podman run --rm -v /path/to/your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign Error: error checking path "/path/to/your_config.fcc": stat /path/to/your_config.fcc: no such file or directory ``` --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index fb95b4816..b43e6b906 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ podman pull quay.io/coreos/fcct:v0.2.0 podman run -it --rm quay.io/coreos/fcct:v0.2.0 -pretty -strict < your_config.fcc > transpiled_config.ign # Run fcct using files. -podman run --rm -v your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign +podman run --rm -v /path/to/your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign ``` ### Writing and using Fedora CoreOS Configs From 0d7c2c80dfcdf939dba5d781d4176f8b781e3c2d Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 23 Aug 2019 10:18:28 -0700 Subject: [PATCH 047/877] main: use O_TRUNC when opening output file Ensure we're not just overwriting the first bit of the file but rather are replacing it. --- internal/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/main.go b/internal/main.go index 16a9fed68..12cfd8a77 100644 --- a/internal/main.go +++ b/internal/main.go @@ -73,7 +73,7 @@ func main() { if output != "" { var err error - outfile, err = os.OpenFile(output, os.O_WRONLY|os.O_CREATE, 0644) + outfile, err = os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { fail("failed to open %s: %v\n", output, err) } From 20ae6f6a53189a4ab561adf3043c9bc5e0f3776d Mon Sep 17 00:00:00 2001 From: nicolas Trauwaen Date: Fri, 30 Aug 2019 17:04:05 +0200 Subject: [PATCH 048/877] Removed -t flag for podman run using stdin As said in the (libpod doc)[https://github.com/containers/libpod/blob/master/docs/podman-run.1.md] > NOTE: The -t option is incompatible with a redirection of the podman client standard input. If you leave the ```-t``` flag your_config.fcc content is sent to stdout and the prompt is waiting endlessly. --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index b43e6b906..931b4bf09 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -28,7 +28,7 @@ This example uses podman, but docker can also be used. Substitute v0.2.0 with th podman pull quay.io/coreos/fcct:v0.2.0 # Run fcct using standard in and standard out -podman run -it --rm quay.io/coreos/fcct:v0.2.0 -pretty -strict < your_config.fcc > transpiled_config.ign +podman run -i --rm quay.io/coreos/fcct:v0.2.0 -pretty -strict < your_config.fcc > transpiled_config.ign # Run fcct using files. podman run --rm -v /path/to/your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign From 5bfe686b25f19849a52a0f481077e4d85acb44b3 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 30 Sep 2019 15:33:46 -0700 Subject: [PATCH 049/877] *: return reports, print reports to stderr Do not print from the translation functions, instead return reports. Print those reports to stderr from where translate was called. --- config/config.go | 19 ++++++++++++------- config/v1_0/fcos.go | 19 ++++++++++--------- internal/main.go | 3 ++- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index fcf18be8c..1dfb71c42 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,7 @@ import ( "github.com/coreos/fcct/config/v1_0" "github.com/coreos/go-semver/semver" + "github.com/coreos/vcontext/report" "gopkg.in/yaml.v3" ) @@ -42,29 +43,33 @@ func getTranslator(variant string, version semver.Version) (translator, error) { return t, nil } -type translator func([]byte, common.TranslateOptions) ([]byte, error) +// translators take a raw config and translate it to a raw Ignition config. The report returned should include any +// errors, warnings, etc and may or may not be fatal. If report is fatal, or other errors are encountered while translating +// translators should return an error. +type translator func([]byte, common.TranslateOptions) ([]byte, report.Report, error) -// Translate wraps all of the actual translate functions in a switch that determines the correct one to call -func Translate(input []byte, options common.TranslateOptions) ([]byte, error) { +// Translate wraps all of the actual translate functions in a switch that determines the correct one to call. +// Translate returns an error if the report had fatal errors or if other errors occured during translation. +func Translate(input []byte, options common.TranslateOptions) ([]byte, report.Report, error) { // first determine version. This will ignore most fields, so don't use strict ver := common.Common{} if err := yaml.Unmarshal(input, &ver); err != nil { - return nil, fmt.Errorf("Error unmarshaling yaml: %v", err) + return nil, report.Report{}, fmt.Errorf("Error unmarshaling yaml: %v", err) } if ver.Variant == "" { - return nil, ErrNoVariant + return nil, report.Report{}, ErrNoVariant } tmp, err := semver.NewVersion(ver.Version) if err != nil { - return nil, ErrInvalidVersion + return nil, report.Report{}, ErrInvalidVersion } version := *tmp translator, err := getTranslator(ver.Variant, version) if err != nil { - return nil, err + return nil, report.Report{}, err } return translator(input, options) diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index 623e58bf6..793df02e3 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -16,7 +16,6 @@ package v1_0 import ( "errors" - "fmt" "reflect" base_0_1 "github.com/coreos/fcct/base/v0_1" @@ -55,12 +54,15 @@ func (c Config) Translate() (types.Config, error) { return v3_0.Merge(distro, base), nil } -func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, error) { +// TranslateBytes translates from a v1.0 fcc to a v3.0.0 Ignition config. It returns a report of any errors or +// warnings in the source and resultant config. If the report has fatal errors or it encounters other problems +// translating, an error is returned. +func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, report.Report, error) { cfg := Config{} contextTree, err := common.Unmarshal(input, &cfg, options.Strict) if err != nil { - return nil, err + return nil, report.Report{}, err } r := validate.Validate(cfg, "yaml") @@ -70,24 +72,23 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, erro r.Merge(validate.ValidateCustom(cfg, "yaml", unusedKeyCheck)) r.Correlate(contextTree) if r.IsFatal() { - fmt.Println(r.String()) - return nil, ErrInvalidConfig + return nil, r, ErrInvalidConfig } final, err := cfg.Translate() if err != nil { - return nil, err + return nil, r, err } translatedTree := common.ToCamelCase(contextTree) second := validate.Validate(final, "json") second.Correlate(translatedTree) r.Merge(second) - fmt.Println(r.String()) if r.IsFatal() { - return nil, ErrInvalidConfig + return nil, r, ErrInvalidConfig } - return common.Marshal(final, options.Pretty) + outbytes, err := common.Marshal(final, options.Pretty) + return outbytes, r, err } diff --git a/internal/main.go b/internal/main.go index 12cfd8a77..e02702e25 100644 --- a/internal/main.go +++ b/internal/main.go @@ -66,7 +66,8 @@ func main() { fail("failed to read %s: %v\n", infile.Name(), err) } - dataOut, err := config.Translate(dataIn, options) + dataOut, r, err := config.Translate(dataIn, options) + fmt.Fprintf(os.Stderr, "%s", r.String()) if err != nil { fail("Error translating config: %v\n", err) } From a338b6b67fad25c304d94d726a1fbeb4a1e96afd Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 30 Sep 2019 15:38:39 -0700 Subject: [PATCH 050/877] docs: document inline file contents --- docs/configuration-v1_0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration-v1_0.md b/docs/configuration-v1_0.md index 6bcbce42c..6c6a41a33 100644 --- a/docs/configuration-v1_0.md +++ b/docs/configuration-v1_0.md @@ -55,7 +55,8 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `source` must be specified if `overwrite` is true. Defaults to false. * **_contents_** (object): options related to the contents of the file. * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. - * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created. + * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created. Mutually exclusive with `inline`. + * **_inline_** (string): the contents of the file. Mutually exclusive with `source`. * **_verification_** (object): options related to the verification of the file contents. * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. * **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents` From 55b145b8ffea265b125baf0b91412eac5b634bd6 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 3 Oct 2019 15:49:58 -0700 Subject: [PATCH 051/877] distro: add to config, do not create a new one Distro sections should consume the configs generated from base sections. --- config/v1_0/fcos.go | 7 +++---- distro/fcos/v0_1/translate.go | 9 +++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index 793df02e3..a1ec08725 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -22,7 +22,6 @@ import ( "github.com/coreos/fcct/config/common" fcos_0_1 "github.com/coreos/fcct/distro/fcos/v0_1" - "github.com/coreos/ignition/v2/config/v3_0" "github.com/coreos/ignition/v2/config/v3_0/types" ignvalidate "github.com/coreos/ignition/v2/config/validate" "github.com/coreos/vcontext/path" @@ -41,17 +40,17 @@ type Config struct { } func (c Config) Translate() (types.Config, error) { - base, err := c.Config.ToIgn3_0() + cfg, err := c.Config.ToIgn3_0() if err != nil { return types.Config{}, err } - distro, err := c.Fcos.ToIgn3_0() + cfg, err = c.Fcos.ToIgn3_0(cfg) if err != nil { return types.Config{}, err } - return v3_0.Merge(distro, base), nil + return cfg, nil } // TranslateBytes translates from a v1.0 fcc to a v3.0.0 Ignition config. It returns a report of any errors or diff --git a/distro/fcos/v0_1/translate.go b/distro/fcos/v0_1/translate.go index 0f6188ee5..ad45bd9d1 100644 --- a/distro/fcos/v0_1/translate.go +++ b/distro/fcos/v0_1/translate.go @@ -18,10 +18,7 @@ import ( "github.com/coreos/ignition/v2/config/v3_0/types" ) -func (f Fcos) ToIgn3_0() (types.Config, error) { - return types.Config{ - Ignition: types.Ignition{ - Version: "3.0.0", - }, - }, nil +// ToIgn3_0 takes a config and merges in the distro specific bits. +func (f Fcos) ToIgn3_0(in types.Config) (types.Config, error) { + return in, nil } From 9bee2719decb3d96297e8bd197100db473aa07a1 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 3 Oct 2019 15:57:21 -0700 Subject: [PATCH 052/877] translate: add translate from ignition --- go.mod | 1 + translate/tests/pkga/types.go | 36 ++++++ translate/tests/pkgb/types.go | 37 ++++++ translate/tests/readme.txt | 3 + translate/translate.go | 187 +++++++++++++++++++++++++++++++ translate/translate_test.go | 205 ++++++++++++++++++++++++++++++++++ 6 files changed, 469 insertions(+) create mode 100644 translate/tests/pkga/types.go create mode 100644 translate/tests/pkgb/types.go create mode 100644 translate/tests/readme.txt create mode 100644 translate/translate.go create mode 100644 translate/translate_test.go diff --git a/go.mod b/go.mod index 18632dda0..06fa513f4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/coreos/ignition/v2 v2.0.1 github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a github.com/davecgh/go-spew v1.1.1 // indirect + github.com/stretchr/testify v1.3.0 github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae ) diff --git a/translate/tests/pkga/types.go b/translate/tests/pkga/types.go new file mode 100644 index 000000000..20e1ab069 --- /dev/null +++ b/translate/tests/pkga/types.go @@ -0,0 +1,36 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pkga + +type Trivial struct { + A string + B int + C bool +} + +type Nested struct { + D string + Trivial +} + +type TrivialReordered struct { + B int + A string + C bool +} + +type HasList struct { + L []Trivial +} diff --git a/translate/tests/pkgb/types.go b/translate/tests/pkgb/types.go new file mode 100644 index 000000000..eeff6b020 --- /dev/null +++ b/translate/tests/pkgb/types.go @@ -0,0 +1,37 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pkgb + +type Trivial struct { + A string + B int + C bool +} + +type Nested struct { + D string + Trivial +} + +// note: struct ordering is different from pkga +type TrivialReordered struct { + A string + B int + C bool +} + +type HasList struct { + L []Nested +} diff --git a/translate/tests/readme.txt b/translate/tests/readme.txt new file mode 100644 index 000000000..7520837a3 --- /dev/null +++ b/translate/tests/readme.txt @@ -0,0 +1,3 @@ +Tests for this translator are in their own package since it needs to test +translating from one package to another. This pattern should not be replicated +in other places in the codebase diff --git a/translate/translate.go b/translate/translate.go new file mode 100644 index 000000000..05d8ce2ca --- /dev/null +++ b/translate/translate.go @@ -0,0 +1,187 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translate + +import ( + "fmt" + "reflect" + + "github.com/coreos/ignition/v2/config/util" +) + +/* + * This is an automatic translator that replace boilerplate code to copy one + * struct into a nearly identical struct in another package. To use it first + * call NewTranslator() to get a translator instance. This can then have + * additional translation rules (in the form of functions) to translate from + * types in one struct to the other. Those functions are in the form: + * func(typeFromInputStruct) -> typeFromOutputStruct + * These can be closures that reference the translator as well. This allows for + * manually translating some fields but resuming automatic translation on the + * other fields through the Translator.Translate() function. + */ + +// Returns if this type can be translated without a custom translator. Children or other +// ancestors might require custom translators however +func (t translator) translatable(t1, t2 reflect.Type) bool { + k1 := t1.Kind() + k2 := t2.Kind() + if k1 != k2 { + return false + } + switch { + case util.IsPrimitive(k1): + return true + case util.IsInvalidInConfig(k1): + panic(fmt.Sprintf("Encountered invalid kind %s in config. This is a bug, please file a report", k1)) + case k1 == reflect.Ptr || k1 == reflect.Slice: + return t.translatable(t1.Elem(), t2.Elem()) || t.hasTranslator(t1.Elem(), t2.Elem()) + case k1 == reflect.Struct: + return t.translatableStruct(t1, t2) + default: + panic(fmt.Sprintf("Encountered unknown kind %s in config. This is a bug, please file a report", k1)) + } +} + +// precondition: t1, t2 are both of Kind 'struct' +func (t translator) translatableStruct(t1, t2 reflect.Type) bool { + if t1.NumField() != t2.NumField() || t1.Name() != t2.Name() { + return false + } + for i := 0; i < t1.NumField(); i++ { + t1f := t1.Field(i) + t2f, ok := t2.FieldByName(t1f.Name) + + if !ok { + return false + } + if !t.translatable(t1f.Type, t2f.Type) && !t.hasTranslator(t1f.Type, t2f.Type) { + return false + } + } + return true +} + +// checks that t could reasonably be the type of a translator function +func couldBeValidTranslator(t reflect.Type) bool { + if t.Kind() != reflect.Func { + return false + } + if t.NumIn() != 1 || t.NumOut() != 1 { + return false + } + if util.IsInvalidInConfig(t.In(0).Kind()) || util.IsInvalidInConfig(t.Out(0).Kind()) { + return false + } + return true +} + +// translate from one type to another, but deep copy all data +// precondition: vFrom and vTo are the same type as defined by translatable() +// precondition: vTo is addressable and settable +func (t translator) translateSameType(vFrom, vTo reflect.Value) { + k := vFrom.Kind() + switch { + case util.IsPrimitive(k): + // Use convert, even if not needed; type alias to primitives are not + // directly assignable and calling Convert on primitives does no harm + vTo.Set(vFrom.Convert(vTo.Type())) + case k == reflect.Ptr: + if vFrom.IsNil() { + return + } + vTo.Set(reflect.New(vTo.Type().Elem())) + t.translate(vFrom.Elem(), vTo.Elem()) + case k == reflect.Slice: + if vFrom.IsNil() { + return + } + vTo.Set(reflect.MakeSlice(vTo.Type(), vFrom.Len(), vFrom.Len())) + for i := 0; i < vFrom.Len(); i++ { + t.translate(vFrom.Index(i), vTo.Index(i)) + } + case k == reflect.Struct: + for i := 0; i < vFrom.NumField(); i++ { + t.translate(vFrom.Field(i), vTo.FieldByName(vFrom.Type().Field(i).Name)) + } + default: + panic("Encountered types that are not the same when they should be. This is a bug, please file a report") + } +} + +// helper to return if a custom translator was defined +func (t translator) hasTranslator(tFrom, tTo reflect.Type) bool { + return t.getTranslator(tFrom, tTo).IsValid() +} + +// vTo must be addressable, should be acquired by calling reflect.ValueOf() on a variable of the correct type +func (t translator) translate(vFrom, vTo reflect.Value) { + tFrom := vFrom.Type() + tTo := vTo.Type() + if fnv := t.getTranslator(tFrom, tTo); fnv.IsValid() { + vTo.Set(fnv.Call([]reflect.Value{vFrom})[0]) + return + } + if t.translatable(tFrom, tTo) { + t.translateSameType(vFrom, vTo) + return + } + + panic(fmt.Sprintf("Translator not defined for %v to %v", tFrom, tTo)) +} + +type Translator interface { + AddCustomTranslator(t interface{}) + Translate(from, to interface{}) +} + +func NewTranslator() Translator { + return &translator{} +} + +type translator struct { + // List of custom translation funcs, must pass couldBeValidTranslator + // This is only for fields that cannot or should not be trivially translated, + // All trivially translated fields use the default behavior. + translators []reflect.Value +} + +func (t *translator) AddCustomTranslator(fn interface{}) { + fnv := reflect.ValueOf(fn) + if !couldBeValidTranslator(fnv.Type()) { + panic("Tried to register invalid translator function") + } + t.translators = append(t.translators, fnv) +} + +func (t translator) getTranslator(from, to reflect.Type) reflect.Value { + for _, fn := range t.translators { + if fn.Type().In(0) == from && fn.Type().Out(0) == to { + return fn + } + } + return reflect.Value{} +} + +func (t translator) Translate(from, to interface{}) { + fv := reflect.ValueOf(from) + tv := reflect.ValueOf(to) + if fv.Kind() != reflect.Ptr || tv.Kind() != reflect.Ptr { + panic("Translate needs to be called on pointers") + } + fv = fv.Elem() + tv = tv.Elem() + t.translate(fv, tv) +} diff --git a/translate/translate_test.go b/translate/translate_test.go new file mode 100644 index 000000000..5d1adda9c --- /dev/null +++ b/translate/translate_test.go @@ -0,0 +1,205 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/coreos/ignition/v2/config/translate/tests/pkga" + "github.com/coreos/ignition/v2/config/translate/tests/pkgb" +) + +// Note: we need different input and output types which unfortunately means a lot of tests + +func TestTranslateTrivial(t *testing.T) { + in := pkga.Trivial{ + A: "asdf", + B: 5, + C: true, + } + + expected := pkgb.Trivial{ + A: "asdf", + B: 5, + C: true, + } + + got := pkgb.Trivial{} + + trans := NewTranslator() + + trans.Translate(&in, &got) + assert.Equal(t, got, expected, "bad translation") +} + +func TestTranslateNested(t *testing.T) { + in := pkga.Nested{ + D: "foobar", + Trivial: pkga.Trivial{ + A: "asdf", + B: 5, + C: true, + }, + } + + expected := pkgb.Nested{ + D: "foobar", + Trivial: pkgb.Trivial{ + A: "asdf", + B: 5, + C: true, + }, + } + + got := pkgb.Nested{} + + trans := NewTranslator() + + trans.Translate(&in, &got) + assert.Equal(t, got, expected, "bad translation") +} + +func TestTranslateTrivialReordered(t *testing.T) { + in := pkga.TrivialReordered{ + A: "asdf", + B: 5, + C: true, + } + + expected := pkgb.TrivialReordered{ + A: "asdf", + B: 5, + C: true, + } + + got := pkgb.TrivialReordered{} + + trans := NewTranslator() + + trans.Translate(&in, &got) + assert.Equal(t, got, expected, "bad translation") +} + +func TestCustomTranslatorTrivial(t *testing.T) { + tr := func(a pkga.Trivial) pkgb.Nested { + return pkgb.Nested{ + Trivial: pkgb.Trivial{ + A: a.A, + B: a.B, + C: a.C, + }, + D: "abc", + } + } + in := pkga.Trivial{ + A: "asdf", + B: 5, + C: true, + } + + expected := pkgb.Nested{ + D: "abc", + Trivial: pkgb.Trivial{ + A: "asdf", + B: 5, + C: true, + }, + } + + got := pkgb.Nested{} + + trans := NewTranslator() + trans.AddCustomTranslator(tr) + + trans.Translate(&in, &got) + assert.Equal(t, got, expected, "bad translation") +} + +func TestCustomTranslatorTrivialWithAutomaticResume(t *testing.T) { + trans := NewTranslator() + tr := func(a pkga.Trivial) pkgb.Nested { + ret := pkgb.Nested{ + D: "abc", + } + trans.Translate(&a, &ret.Trivial) + return ret + } + in := pkga.Trivial{ + A: "asdf", + B: 5, + C: true, + } + + expected := pkgb.Nested{ + D: "abc", + Trivial: pkgb.Trivial{ + A: "asdf", + B: 5, + C: true, + }, + } + + got := pkgb.Nested{} + + trans.AddCustomTranslator(tr) + + trans.Translate(&in, &got) + assert.Equal(t, got, expected, "bad translation") +} + +func TestCustomTranslatorList(t *testing.T) { + tr := func(a pkga.Trivial) pkgb.Nested { + return pkgb.Nested{ + Trivial: pkgb.Trivial{ + A: a.A, + B: a.B, + C: a.C, + }, + D: "abc", + } + } + in := pkga.HasList{ + L: []pkga.Trivial{ + { + A: "asdf", + B: 5, + C: true, + }, + }, + } + + expected := pkgb.HasList{ + L: []pkgb.Nested{ + { + D: "abc", + Trivial: pkgb.Trivial{ + A: "asdf", + B: 5, + C: true, + }, + }, + }, + } + + got := pkgb.HasList{} + + trans := NewTranslator() + trans.AddCustomTranslator(tr) + + trans.Translate(&in, &got) + assert.Equal(t, got, expected, "bad translation") +} From 692fa1e2a668cf89e28f934643071add156a3fe1 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 7 Oct 2019 10:18:38 -0700 Subject: [PATCH 053/877] translate: keep track of translations Add a mechanism to keep track of what changed. Auto-derive paths like we do when validating. Require custom translation functions to also return a list of translations. --- translate/translate.go | 164 ++++++++++++++++++++++++++++++++---- translate/translate_test.go | 115 ++++++++++++++++++++----- 2 files changed, 242 insertions(+), 37 deletions(-) diff --git a/translate/translate.go b/translate/translate.go index 05d8ce2ca..d94476fc5 100644 --- a/translate/translate.go +++ b/translate/translate.go @@ -17,8 +17,10 @@ package translate import ( "fmt" "reflect" + "strings" "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/vcontext/path" ) /* @@ -33,6 +35,10 @@ import ( * other fields through the Translator.Translate() function. */ +var ( + translationsType = reflect.TypeOf(TranslationSet{}) +) + // Returns if this type can be translated without a custom translator. Children or other // ancestors might require custom translators however func (t translator) translatable(t1, t2 reflect.Type) bool { @@ -79,42 +85,69 @@ func couldBeValidTranslator(t reflect.Type) bool { if t.Kind() != reflect.Func { return false } - if t.NumIn() != 1 || t.NumOut() != 1 { + if t.NumIn() != 1 || t.NumOut() != 2 { return false } - if util.IsInvalidInConfig(t.In(0).Kind()) || util.IsInvalidInConfig(t.Out(0).Kind()) { + if util.IsInvalidInConfig(t.In(0).Kind()) || + util.IsInvalidInConfig(t.Out(0).Kind()) || + t.Out(1) != translationsType { return false } return true } +// fieldName returns the name uses when (un)marshalling a field. t should be a reflect.Value of a struct, +// index is the field index, and tag is the struct tag used when (un)marshalling (e.g. "json" or "yaml") +func fieldName(t reflect.Value, index int, tag string) string { + f := t.Type().Field(index) + if tag == "" { + return f.Name + } + return strings.Split(f.Tag.Get(tag), ",")[0] +} + // translate from one type to another, but deep copy all data // precondition: vFrom and vTo are the same type as defined by translatable() // precondition: vTo is addressable and settable -func (t translator) translateSameType(vFrom, vTo reflect.Value) { +func (t translator) translateSameType(vFrom, vTo reflect.Value, fromPath, toPath path.ContextPath) { k := vFrom.Kind() switch { case util.IsPrimitive(k): // Use convert, even if not needed; type alias to primitives are not // directly assignable and calling Convert on primitives does no harm vTo.Set(vFrom.Convert(vTo.Type())) + t.translations.AddTranslation(fromPath, toPath) case k == reflect.Ptr: if vFrom.IsNil() { return } vTo.Set(reflect.New(vTo.Type().Elem())) - t.translate(vFrom.Elem(), vTo.Elem()) + t.translate(vFrom.Elem(), vTo.Elem(), fromPath, toPath) case k == reflect.Slice: if vFrom.IsNil() { return } vTo.Set(reflect.MakeSlice(vTo.Type(), vFrom.Len(), vFrom.Len())) for i := 0; i < vFrom.Len(); i++ { - t.translate(vFrom.Index(i), vTo.Index(i)) + t.translate(vFrom.Index(i), vTo.Index(i), fromPath.Append(i), toPath.Append(i)) } case k == reflect.Struct: for i := 0; i < vFrom.NumField(); i++ { - t.translate(vFrom.Field(i), vTo.FieldByName(vFrom.Type().Field(i).Name)) + fieldGoName := vFrom.Type().Field(i).Name + toStructField, ok := vTo.Type().FieldByName(fieldGoName) + if !ok { + panic("vTo did not have a matching type. This is a bug; please file a report") + } + toFieldIndex := toStructField.Index[0] + vToField := vTo.FieldByName(fieldGoName) + + from := fromPath.Append(fieldName(vFrom, i, fromPath.Tag)) + to := toPath.Append(fieldName(vTo, toFieldIndex, toPath.Tag)) + if vFrom.Type().Field(i).Anonymous { + from = fromPath + to = toPath + } + t.translate(vFrom.Field(i), vToField, from, to) } default: panic("Encountered types that are not the same when they should be. This is a bug, please file a report") @@ -127,15 +160,24 @@ func (t translator) hasTranslator(tFrom, tTo reflect.Type) bool { } // vTo must be addressable, should be acquired by calling reflect.ValueOf() on a variable of the correct type -func (t translator) translate(vFrom, vTo reflect.Value) { +func (t translator) translate(vFrom, vTo reflect.Value, fromPath, toPath path.ContextPath) { tFrom := vFrom.Type() tTo := vTo.Type() if fnv := t.getTranslator(tFrom, tTo); fnv.IsValid() { - vTo.Set(fnv.Call([]reflect.Value{vFrom})[0]) + returns := fnv.Call([]reflect.Value{vFrom}) + vTo.Set(returns[0]) + + // handle all the translations and "rebase" them to our current place + retSet := returns[1].Interface().(TranslationSet) + for _, trans := range retSet.Set { + from := fromPath.Append(trans.From.Path...) + to := toPath.Append(trans.To.Path...) + t.translations.AddTranslation(from, to) + } return } if t.translatable(tFrom, tTo) { - t.translateSameType(vFrom, vTo) + t.translateSameType(vFrom, vTo, fromPath, toPath) return } @@ -143,21 +185,105 @@ func (t translator) translate(vFrom, vTo reflect.Value) { } type Translator interface { + // Adds a custom translator for cases where the structs are not identical. Must be of type + // func(fromType) -> (toType, TranslationSet). The translator should return the set of all + // translations it did. AddCustomTranslator(t interface{}) - Translate(from, to interface{}) + // Also returns a list of source and dest paths, autocompleted by fromTag and toTag + Translate(from, to interface{}) TranslationSet +} + +// Translation represents how a path changes when translating. If something at $yaml.storage.filesystems.4 +// generates content at $json.systemd.units.3 a translation can represent that. This allows validation errors +// in Ignition structs to be tracked back to their source in the yaml. +type Translation struct { + From path.ContextPath + To path.ContextPath +} + +// TranslationSet represents all of the translations that occurred. They're stored in a map from a string representation +// of the destination path to the translation struct. The map is purely an optimization to allow fast lookups. Ideally the +// map would just be from the destination path.ContextPath to the source path.ContextPath, but ContextPath contains a slice +// which are not comparable and thus cannot be used as keys in maps. +type TranslationSet struct { + FromTag string + ToTag string + Set map[string]Translation +} + +// AddTranslation adds a translation to the set +func (ts TranslationSet) AddTranslation(from, to path.ContextPath) { + translation := Translation{ + From: from, + To: to, + } + toString := translation.To.String() + ts.Set[toString] = translation } -func NewTranslator() Translator { - return &translator{} +// Shortcut for AddTranslation for identity translations +func (ts TranslationSet) AddIdentity(paths ...string) { + for _, p := range paths { + from := path.New(ts.FromTag, p) + to := path.New(ts.ToTag, p) + ts.AddTranslation(from, to) + } +} + +// Merge adds all the entries to the set. It mutates the Set in place. +func (ts TranslationSet) Merge(from TranslationSet) { + for _, t := range from.Set { + ts.Set[t.From.String()] = t + } +} + +// MergeP is like Merge, but first it calls Prefix on the set being merged in. +func (ts TranslationSet) MergeP(prefix interface{}, from TranslationSet) { + from = from.Prefix(prefix) + for _, t := range from.Set { + ts.Set[t.To.String()] = t + } +} + +// Prefix returns a TranslationSet with all translation paths prefixed by prefix. +func (ts TranslationSet) Prefix(prefix interface{}) TranslationSet { + ret := TranslationSet{ + FromTag: ts.FromTag, + ToTag: ts.ToTag, + Set: map[string]Translation{}, + } + p := []interface{}{prefix} + for _, tr := range ts.Set { + tr.From.Path = append(p, tr.From.Path...) + tr.To.Path = append(p, tr.To.Path...) + ret.AddTranslation(tr.From, tr.To) + } + return ret +} + +// NewTranslator creates a new Translator for translating from types with fromTag struct tags (e.g. "yaml") +// to types with toTag struct tages (e.g. "json"). These tags are used when determining paths when generating +// the TranslationSet returned by Translator.Translate() +func NewTranslator(fromTag, toTag string) Translator { + return &translator{ + translations: TranslationSet{ + FromTag: fromTag, + ToTag: toTag, + Set: map[string]Translation{}, + }, + } } type translator struct { // List of custom translation funcs, must pass couldBeValidTranslator // This is only for fields that cannot or should not be trivially translated, // All trivially translated fields use the default behavior. - translators []reflect.Value + translators []reflect.Value + translations TranslationSet } +// fn should be of the form func(fromType, translationsMap) -> toType +// fn should mutate translationsMap to add all the translations it did func (t *translator) AddCustomTranslator(fn interface{}) { fnv := reflect.ValueOf(fn) if !couldBeValidTranslator(fnv.Type()) { @@ -175,7 +301,8 @@ func (t translator) getTranslator(from, to reflect.Type) reflect.Value { return reflect.Value{} } -func (t translator) Translate(from, to interface{}) { +// Translate translates from into to and returns a set of all the path changes it performed. +func (t translator) Translate(from, to interface{}) TranslationSet { fv := reflect.ValueOf(from) tv := reflect.ValueOf(to) if fv.Kind() != reflect.Ptr || tv.Kind() != reflect.Ptr { @@ -183,5 +310,12 @@ func (t translator) Translate(from, to interface{}) { } fv = fv.Elem() tv = tv.Elem() - t.translate(fv, tv) + // Make sure to clear this every time` + t.translations = TranslationSet{ + FromTag: t.translations.FromTag, + ToTag: t.translations.ToTag, + Set: map[string]Translation{}, + } + t.translate(fv, tv, path.New(t.translations.FromTag), path.New(t.translations.ToTag)) + return t.translations } diff --git a/translate/translate_test.go b/translate/translate_test.go index 5d1adda9c..a6396eadf 100644 --- a/translate/translate_test.go +++ b/translate/translate_test.go @@ -17,14 +17,34 @@ package translate import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/coreos/fcct/translate/tests/pkga" + "github.com/coreos/fcct/translate/tests/pkgb" - "github.com/coreos/ignition/v2/config/translate/tests/pkga" - "github.com/coreos/ignition/v2/config/translate/tests/pkgb" + "github.com/coreos/vcontext/path" + "github.com/stretchr/testify/assert" ) // Note: we need different input and output types which unfortunately means a lot of tests +// mkTrans makes a TranslationSet with no tag in the paths consuming pairs of args. i.e: +// mkTrans(from1, to1, from2, to2) -> a set wiht from1->to1, from2->to2 +// This is just a shorthand for making writing tests easier +func mkTrans(paths ...path.ContextPath) TranslationSet { + ret := TranslationSet{Set: map[string]Translation{}} + if len(paths)%2 == 1 { + panic("Odd number of args to mkTrans") + } + for i := 0; i < len(paths); i += 2 { + ret.AddTranslation(paths[i], paths[i+1]) + } + return ret +} + +// fp means "fastpath"; super shorthand, we'll use it a lot +func fp(parts ...interface{}) path.ContextPath { + return path.New("", parts...) +} + func TestTranslateTrivial(t *testing.T) { in := pkga.Trivial{ A: "asdf", @@ -37,13 +57,19 @@ func TestTranslateTrivial(t *testing.T) { B: 5, C: true, } + exTrans := mkTrans( + fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + ) got := pkgb.Trivial{} - trans := NewTranslator() + trans := NewTranslator("", "") - trans.Translate(&in, &got) + ts := trans.Translate(&in, &got) assert.Equal(t, got, expected, "bad translation") + assert.Equal(t, ts, exTrans, "bad translation") } func TestTranslateNested(t *testing.T) { @@ -64,13 +90,20 @@ func TestTranslateNested(t *testing.T) { C: true, }, } + exTrans := mkTrans( + fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + fp("D"), fp("D"), + ) got := pkgb.Nested{} - trans := NewTranslator() + trans := NewTranslator("", "") - trans.Translate(&in, &got) + ts := trans.Translate(&in, &got) assert.Equal(t, got, expected, "bad translation") + assert.Equal(t, ts, exTrans, "bad translation") } func TestTranslateTrivialReordered(t *testing.T) { @@ -85,17 +118,28 @@ func TestTranslateTrivialReordered(t *testing.T) { B: 5, C: true, } + exTrans := mkTrans( + fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + ) got := pkgb.TrivialReordered{} - trans := NewTranslator() + trans := NewTranslator("", "") - trans.Translate(&in, &got) + ts := trans.Translate(&in, &got) assert.Equal(t, got, expected, "bad translation") + assert.Equal(t, ts, exTrans, "bad translation") } func TestCustomTranslatorTrivial(t *testing.T) { - tr := func(a pkga.Trivial) pkgb.Nested { + tr := func(a pkga.Trivial) (pkgb.Nested, TranslationSet) { + ts := mkTrans(fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + fp("C"), fp("D"), + ) return pkgb.Nested{ Trivial: pkgb.Trivial{ A: a.A, @@ -103,7 +147,7 @@ func TestCustomTranslatorTrivial(t *testing.T) { C: a.C, }, D: "abc", - } + }, ts } in := pkga.Trivial{ A: "asdf", @@ -119,30 +163,44 @@ func TestCustomTranslatorTrivial(t *testing.T) { C: true, }, } + exTrans := mkTrans( + fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + fp("C"), fp("D"), + ) got := pkgb.Nested{} - trans := NewTranslator() + trans := NewTranslator("", "") trans.AddCustomTranslator(tr) - trans.Translate(&in, &got) + ts := trans.Translate(&in, &got) assert.Equal(t, got, expected, "bad translation") + assert.Equal(t, ts, exTrans, "bad translation") } func TestCustomTranslatorTrivialWithAutomaticResume(t *testing.T) { - trans := NewTranslator() - tr := func(a pkga.Trivial) pkgb.Nested { + trans := NewTranslator("", "") + tr := func(a pkga.Trivial) (pkgb.Nested, TranslationSet) { ret := pkgb.Nested{ D: "abc", } - trans.Translate(&a, &ret.Trivial) - return ret + ts := trans.Translate(&a, &ret.Trivial) + ts.AddTranslation(fp("C"), fp("D")) + return ret, ts } in := pkga.Trivial{ A: "asdf", B: 5, C: true, } + exTrans := mkTrans( + fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + fp("C"), fp("D"), + ) expected := pkgb.Nested{ D: "abc", @@ -157,12 +215,18 @@ func TestCustomTranslatorTrivialWithAutomaticResume(t *testing.T) { trans.AddCustomTranslator(tr) - trans.Translate(&in, &got) + ts := trans.Translate(&in, &got) assert.Equal(t, got, expected, "bad translation") + assert.Equal(t, ts, exTrans, "bad translation") } func TestCustomTranslatorList(t *testing.T) { - tr := func(a pkga.Trivial) pkgb.Nested { + tr := func(a pkga.Trivial) (pkgb.Nested, TranslationSet) { + ts := mkTrans(fp("A"), fp("A"), + fp("B"), fp("B"), + fp("C"), fp("C"), + fp("C"), fp("D"), + ) return pkgb.Nested{ Trivial: pkgb.Trivial{ A: a.A, @@ -170,7 +234,7 @@ func TestCustomTranslatorList(t *testing.T) { C: a.C, }, D: "abc", - } + }, ts } in := pkga.HasList{ L: []pkga.Trivial{ @@ -194,12 +258,19 @@ func TestCustomTranslatorList(t *testing.T) { }, }, } + exTrans := mkTrans( + fp("L", 0, "A"), fp("L", 0, "A"), + fp("L", 0, "B"), fp("L", 0, "B"), + fp("L", 0, "C"), fp("L", 0, "C"), + fp("L", 0, "C"), fp("L", 0, "D"), + ) got := pkgb.HasList{} - trans := NewTranslator() + trans := NewTranslator("", "") trans.AddCustomTranslator(tr) - trans.Translate(&in, &got) + ts := trans.Translate(&in, &got) assert.Equal(t, got, expected, "bad translation") + assert.Equal(t, ts, exTrans, "bad translation") } From 23146d336be10d7c1d77554ac03f89da60b63300 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 8 Oct 2019 15:05:24 -0700 Subject: [PATCH 054/877] *: use new translation tracking Keep track of how paths change when translating, then use that convert reports from Ignition's validation to have logical paths that match the fcc. --- base/v0_1/translate.go | 62 +++++++++++--------- base/v0_1/translate_test.go | 53 ++++++++++++++--- config/common/common.go | 66 +++++++++++---------- config/common/common_test.go | 107 +++------------------------------- config/v1_0/fcos.go | 21 ++++--- distro/fcos/v0_1/translate.go | 6 +- 6 files changed, 139 insertions(+), 176 deletions(-) diff --git a/base/v0_1/translate.go b/base/v0_1/translate.go index c48f25fb7..dba8c7aee 100644 --- a/base/v0_1/translate.go +++ b/base/v0_1/translate.go @@ -17,75 +17,85 @@ package v0_1 import ( "net/url" - "github.com/coreos/ignition/v2/config/translate" + "github.com/coreos/fcct/translate" + "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/coreos/vcontext/path" "github.com/vincent-petithory/dataurl" ) -func (c Config) ToIgn3_0() (types.Config, error) { +// ToIgn3_0 translates the config to an Ignition config. It also returns the set of translations +// it did so paths in the resultant config can be tracked back to their source in the source config. +func (c Config) ToIgn3_0() (types.Config, translate.TranslationSet, error) { ret := types.Config{} - tr := translate.NewTranslator() + tr := translate.NewTranslator("yaml", "json") tr.AddCustomTranslator(translateIgnition) tr.AddCustomTranslator(translateFile) tr.AddCustomTranslator(translateDirectory) tr.AddCustomTranslator(translateLink) - tr.Translate(&c, &ret) - return ret, nil + translations := tr.Translate(&c, &ret) + return ret, translations, nil } -func translateIgnition(from Ignition) (to types.Ignition) { - tr := translate.NewTranslator() +func translateIgnition(from Ignition) (to types.Ignition, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") to.Version = types.MaxVersion.String() - tr.Translate(&from.Config, &to.Config) - tr.Translate(&from.Security, &to.Security) - tr.Translate(&from.Timeouts, &to.Timeouts) + tm = tr.Translate(&from.Config, &to.Config).Prefix("config") + tm.MergeP("security", tr.Translate(&from.Security, &to.Security)) + tm.MergeP("timeouts", tr.Translate(&from.Timeouts, &to.Timeouts)) return } -func translateFile(from File) (to types.File) { - tr := translate.NewTranslator() +func translateFile(from File) (to types.File, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") tr.AddCustomTranslator(translateFileContents) - tr.Translate(&from.Group, &to.Group) - tr.Translate(&from.User, &to.User) - tr.Translate(&from.Append, &to.Append) - tr.Translate(&from.Contents, &to.Contents) + tm = tr.Translate(&from.Group, &to.Group).Prefix("group") + tm.MergeP("user", tr.Translate(&from.User, &to.User)) + tm.MergeP("append", tr.Translate(&from.Append, &to.Append)) + tm.MergeP("contents", tr.Translate(&from.Contents, &to.Contents)) to.Overwrite = from.Overwrite to.Path = from.Path to.Mode = from.Mode + tm.AddIdentity("overwrite", "path", "mode") return } -func translateFileContents(from FileContents) (to types.FileContents) { +func translateFileContents(from FileContents) (to types.FileContents, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Verification, &to.Verification).Prefix("verification") to.Source = from.Source to.Compression = from.Compression - to.Verification.Hash = from.Verification.Hash + tm.AddIdentity("source", "compression") if from.Inline != nil { src := (&url.URL{ Scheme: "data", Opaque: "," + dataurl.EscapeString(*from.Inline), }).String() to.Source = &src + tm.AddTranslation(path.New("yaml", "inline"), path.New("json", "source")) } return } -func translateDirectory(from Directory) (to types.Directory) { - tr := translate.NewTranslator() - tr.Translate(&from.Group, &to.Group) - tr.Translate(&from.User, &to.User) +func translateDirectory(from Directory) (to types.Directory, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Group, &to.Group).Prefix("group") + tm.MergeP("user", tr.Translate(&from.User, &to.User)) to.Overwrite = from.Overwrite to.Path = from.Path to.Mode = from.Mode + tm.AddIdentity("overwrite", "path", "mode") return } -func translateLink(from Link) (to types.Link) { - tr := translate.NewTranslator() - tr.Translate(&from.Group, &to.Group) - tr.Translate(&from.User, &to.User) +func translateLink(from Link) (to types.Link, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Group, &to.Group).Prefix("group") + tm.MergeP("user", tr.Translate(&from.User, &to.User)) to.Target = from.Target to.Hard = from.Hard to.Overwrite = from.Overwrite to.Path = from.Path + tm.AddIdentity("target", "hard", "overwrite", "path") return } diff --git a/base/v0_1/translate_test.go b/base/v0_1/translate_test.go index c44b6806a..2d890a2a3 100644 --- a/base/v0_1/translate_test.go +++ b/base/v0_1/translate_test.go @@ -18,21 +18,49 @@ import ( "reflect" "testing" + "github.com/coreos/fcct/translate" + "github.com/coreos/ignition/v2/config/util" "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/coreos/vcontext/path" ) // Most of this is covered by the Ignition translator generic tests, so just test the custom bits +// verifyTranslations ensures all the translations are identity, unless they match a listed one +// it returns the offending translation if there is one +func verifyTranslations(set translate.TranslationSet, exceptions ...translate.Translation) *translate.Translation { + exceptionSet := translate.TranslationSet{ + FromTag: set.FromTag, + ToTag: set.ToTag, + Set: map[string]translate.Translation{}, + } + for _, ex := range exceptions { + exceptionSet.AddTranslation(ex.From, ex.To) + } + for key, translation := range set.Set { + if ex, ok := exceptionSet.Set[key]; ok { + if !reflect.DeepEqual(translation, ex) { + return &ex + } + } else if !reflect.DeepEqual(translation.From.Path, translation.To.Path) { + return &translation + } + } + return nil +} + // TestTranslateFile tests translating the ct storage.files.[i] entries to ignition storage.files.[i] entires. func TestTranslateFile(t *testing.T) { tests := []struct { - in File - out types.File + in File + out types.File + exceptions []translate.Translation }{ { File{}, types.File{}, + nil, }, { // contains invalid (by the validator's definition) combinations of fields, @@ -113,14 +141,25 @@ func TestTranslateFile(t *testing.T) { }, }, }, + []translate.Translation{ + { + From: path.New("yaml", "append", 1, "inline"), + To: path.New("json", "append", 1, "source"), + }, + }, }, } for i, test := range tests { - actual := translateFile(test.in) + actual, translations := translateFile(test.in) + if !reflect.DeepEqual(actual, test.out) { t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) } + + if errT := verifyTranslations(translations, test.exceptions...); errT != nil { + t.Errorf("#%d: bad translation: %v", i, *errT) + } } } @@ -171,7 +210,7 @@ func TestTranslateDirectory(t *testing.T) { } for i, test := range tests { - actual := translateDirectory(test.in) + actual, _ := translateDirectory(test.in) if !reflect.DeepEqual(actual, test.out) { t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) } @@ -227,7 +266,7 @@ func TestTranslateLink(t *testing.T) { } for i, test := range tests { - actual := translateLink(test.in) + actual, _ := translateLink(test.in) if !reflect.DeepEqual(actual, test.out) { t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) } @@ -249,7 +288,7 @@ func TestTranslateIgnition(t *testing.T) { }, } for i, test := range tests { - actual := translateIgnition(test.in) + actual, _ := translateIgnition(test.in) if !reflect.DeepEqual(actual, test.out) { t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) } @@ -273,7 +312,7 @@ func TestToIgn3_0(t *testing.T) { }, } for i, test := range tests { - actual, err := test.in.ToIgn3_0() + actual, _, err := test.in.ToIgn3_0() if err != nil { t.Errorf("#%d: got error: %v", i, err) } diff --git a/config/common/common.go b/config/common/common.go index 44b068645..0309eda09 100644 --- a/config/common/common.go +++ b/config/common/common.go @@ -17,13 +17,22 @@ package common import ( "bytes" "encoding/json" + "regexp" "strings" + "github.com/coreos/fcct/translate" + + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" "github.com/coreos/vcontext/tree" vyaml "github.com/coreos/vcontext/yaml" "gopkg.in/yaml.v3" ) +var ( + snakeRe = regexp.MustCompile("([A-Z])") +) + type TranslateOptions struct { Pretty bool Strict bool @@ -55,41 +64,34 @@ func Marshal(from interface{}, pretty bool) ([]byte, error) { return json.Marshal(from) } -// camel takes a snake_case string and converting it to camelCase -func camel(in string) string { - words := strings.Split(in, "_") - for i, word := range words[1:] { - words[i+1] = strings.Title(word) +// snakePath converts a path.ContextPath with camelCase elements and returns the +// same path but with snake_case elements instead +func snakePath(p path.ContextPath) path.ContextPath { + ret := path.New(p.Tag) + for _, part := range p.Path { + if str, ok := part.(string); ok { + ret = ret.Append(snake(str)) + } else { + ret = ret.Append(part) + } } - return strings.Join(words, "") + return ret } -// ToCamelCase converts the keys in a context tree from snake_case to camelCase -func ToCamelCase(t tree.Node) tree.Node { - switch n := t.(type) { - case tree.MapNode: - m := tree.MapNode{ - Children: make(map[string]tree.Node, len(n.Children)), - Keys: make(map[string]tree.Leaf, len(n.Keys)), - Marker: n.Marker, - } - for k, v := range n.Children { - m.Children[camel(k)] = ToCamelCase(v) - } - for k, v := range n.Keys { - m.Keys[camel(k)] = v - } - return m - case tree.SliceNode: - s := tree.SliceNode{ - Children: make([]tree.Node, 0, len(n.Children)), - Marker: n.Marker, - } - for _, v := range n.Children { - s.Children = append(s.Children, ToCamelCase(v)) +// snake converts from camelCase (not CamelCase) to snake_case +func snake(in string) string { + return strings.ToLower(snakeRe.ReplaceAllString(in, "_$1")) +} + +// TranslateReportPaths takes a report from a camelCase json document and a set of translations rules, +// applies those rules and converts all camelCase to snake_case. +func TranslateReportPaths(r *report.Report, ts translate.TranslationSet) { + for i, ent := range r.Entries { + context := ent.Context + if t, ok := ts.Set[context.String()]; ok { + context = t.From } - return s - default: // leaf - return t + context = snakePath(context) + r.Entries[i].Context = context } } diff --git a/config/common/common_test.go b/config/common/common_test.go index a6de2539d..4e4d322c2 100644 --- a/config/common/common_test.go +++ b/config/common/common_test.go @@ -15,13 +15,10 @@ package common import ( - "reflect" "testing" - - "github.com/coreos/vcontext/tree" ) -func TestCamel(t *testing.T) { +func TestSnake(t *testing.T) { tests := []struct { in string out string @@ -32,112 +29,22 @@ func TestCamel(t *testing.T) { "foo", }, { - "snake_case", "snakeCase", + "snake_case", }, { - "long_snake_case", "longSnakeCase", + "long_snake_case", }, { - "camelAlready", - "camelAlready", - }, - } - - for i, test := range tests { - if camel(test.in) != test.out { - t.Errorf("#%d: expected %q got %q", i, test.out, camel(test.in)) - } - } -} - -func TestToCamelCase(t *testing.T) { - tests := []struct { - in tree.Node - out tree.Node - }{ - {}, - { - tree.Leaf{ - Marker: tree.MarkerFromIndices(1, 2), - }, - tree.Leaf{ - Marker: tree.MarkerFromIndices(1, 2), - }, - }, - { - tree.MapNode{ - Marker: tree.MarkerFromIndices(1, 2), - Children: map[string]tree.Node{ - "foo_bar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - Keys: map[string]tree.Leaf{ - "foo_bar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - }, - tree.MapNode{ - Marker: tree.MarkerFromIndices(1, 2), - Children: map[string]tree.Node{ - "fooBar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - Keys: map[string]tree.Leaf{ - "fooBar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - }, - }, - { - tree.SliceNode{ - Marker: tree.MarkerFromIndices(5, 6), - Children: []tree.Node{ - tree.MapNode{ - Marker: tree.MarkerFromIndices(1, 2), - Children: map[string]tree.Node{ - "foo_bar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - Keys: map[string]tree.Leaf{ - "foo_bar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - }, - }, - }, - tree.SliceNode{ - Marker: tree.MarkerFromIndices(5, 6), - Children: []tree.Node{ - tree.MapNode{ - Marker: tree.MarkerFromIndices(1, 2), - Children: map[string]tree.Node{ - "fooBar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - Keys: map[string]tree.Leaf{ - "fooBar": tree.Leaf{ - tree.MarkerFromIndices(3, 4), - }, - }, - }, - }, - }, + "snake_already", + "snake_already", }, } for i, test := range tests { - actual := ToCamelCase(test.in) - if !reflect.DeepEqual(actual, test.out) { - t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + if snake(test.in) != test.out { + t.Errorf("#%d: expected %q got %q", i, test.out, snake(test.in)) } } } diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index a1ec08725..a1631a18c 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -21,6 +21,7 @@ import ( base_0_1 "github.com/coreos/fcct/base/v0_1" "github.com/coreos/fcct/config/common" fcos_0_1 "github.com/coreos/fcct/distro/fcos/v0_1" + "github.com/coreos/fcct/translate" "github.com/coreos/ignition/v2/config/v3_0/types" ignvalidate "github.com/coreos/ignition/v2/config/validate" @@ -39,18 +40,20 @@ type Config struct { fcos_0_1.Fcos `yaml:",inline"` } -func (c Config) Translate() (types.Config, error) { - cfg, err := c.Config.ToIgn3_0() +func (c Config) Translate() (types.Config, translate.TranslationSet, error) { + cfg, baseTranslations, err := c.Config.ToIgn3_0() if err != nil { - return types.Config{}, err + return types.Config{}, translate.TranslationSet{}, err } - cfg, err = c.Fcos.ToIgn3_0(cfg) + finalcfg, distroTranslations, err := c.Fcos.ToIgn3_0(cfg) if err != nil { - return types.Config{}, err + return types.Config{}, translate.TranslationSet{}, err } - return cfg, nil + baseTranslations.Merge(distroTranslations) + + return finalcfg, baseTranslations, nil } // TranslateBytes translates from a v1.0 fcc to a v3.0.0 Ignition config. It returns a report of any errors or @@ -74,14 +77,14 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, repo return nil, r, ErrInvalidConfig } - final, err := cfg.Translate() + final, translations, err := cfg.Translate() if err != nil { return nil, r, err } - translatedTree := common.ToCamelCase(contextTree) second := validate.Validate(final, "json") - second.Correlate(translatedTree) + common.TranslateReportPaths(&second, translations) + second.Correlate(contextTree) r.Merge(second) if r.IsFatal() { diff --git a/distro/fcos/v0_1/translate.go b/distro/fcos/v0_1/translate.go index ad45bd9d1..17918bc0d 100644 --- a/distro/fcos/v0_1/translate.go +++ b/distro/fcos/v0_1/translate.go @@ -15,10 +15,12 @@ package fcos_0_1 import ( + "github.com/coreos/fcct/translate" + "github.com/coreos/ignition/v2/config/v3_0/types" ) // ToIgn3_0 takes a config and merges in the distro specific bits. -func (f Fcos) ToIgn3_0(in types.Config) (types.Config, error) { - return in, nil +func (f Fcos) ToIgn3_0(in types.Config) (types.Config, translate.TranslationSet, error) { + return in, translate.TranslationSet{}, nil } From 6e48035f2530bbdb7cce5959cb485a8a8281b417 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 11 Oct 2019 11:05:09 -0700 Subject: [PATCH 055/877] base/v0_2_exp: add package Add package to track Ignition spec 3.1.0-experimental and to add other new features to. --- base/v0_2_exp/schema.go | 203 ++++++++++++++++++++ base/v0_2_exp/translate.go | 101 ++++++++++ base/v0_2_exp/translate_test.go | 324 ++++++++++++++++++++++++++++++++ base/v0_2_exp/validate.go | 33 ++++ base/v0_2_exp/validate_test.go | 79 ++++++++ 5 files changed, 740 insertions(+) create mode 100644 base/v0_2_exp/schema.go create mode 100644 base/v0_2_exp/translate.go create mode 100644 base/v0_2_exp/translate_test.go create mode 100644 base/v0_2_exp/validate.go create mode 100644 base/v0_2_exp/validate_test.go diff --git a/base/v0_2_exp/schema.go b/base/v0_2_exp/schema.go new file mode 100644 index 000000000..8ed19118a --- /dev/null +++ b/base/v0_2_exp/schema.go @@ -0,0 +1,203 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_2_exp + +type CaReference struct { + Source string `yaml:"source"` + Verification Verification `yaml:"verification"` +} + +type Config struct { + Ignition Ignition `yaml:"ignition"` + Passwd Passwd `yaml:"passwd"` + Storage Storage `yaml:"storage"` + Systemd Systemd `yaml:"systemd"` +} + +type ConfigReference struct { + Source *string `yaml:"source"` + Verification Verification `yaml:"verification"` +} + +type Device string + +type Directory struct { + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Mode *int `yaml:"mode"` +} + +type Disk struct { + Device string `yaml:"device"` + Partitions []Partition `yaml:"partitions"` + WipeTable *bool `yaml:"wipe_table"` +} + +type Dropin struct { + Contents *string `yaml:"contents"` + Name string `yaml:"name"` +} + +type File struct { + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Append []FileContents `yaml:"append"` + Contents FileContents `yaml:"contents"` + Mode *int `yaml:"mode"` +} + +type FileContents struct { + Compression *string `yaml:"compression"` + Source *string `yaml:"source"` + Inline *string `yaml:"inline"` // Added, not in ignition spec + Verification Verification `yaml:"verification"` +} + +type Filesystem struct { + Device string `yaml:"device"` + Format *string `yaml:"format"` + Label *string `yaml:"label"` + Options []FilesystemOption `yaml:"options"` + Path *string `yaml:"path"` + UUID *string `yaml:"uuid"` + WipeFilesystem *bool `yaml:"wipe_filesystem"` +} + +type FilesystemOption string + +type Group string + +type Ignition struct { + Config IgnitionConfig `yaml:"config"` + Security Security `yaml:"security"` + Timeouts Timeouts `yaml:"timeouts"` +} + +type IgnitionConfig struct { + Merge []ConfigReference `yaml:"merge"` + Replace ConfigReference `yaml:"replace"` +} + +type Link struct { + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Hard *bool `yaml:"hard"` + Target string `yaml:"target"` +} + +type NodeGroup struct { + ID *int `yaml:"id"` + Name *string `yaml:"name"` +} + +type NodeUser struct { + ID *int `yaml:"id"` + Name *string `yaml:"name"` +} + +type Partition struct { + GUID *string `yaml:"guid"` + Label *string `yaml:"label"` + Number int `yaml:"number"` + ShouldExist *bool `yaml:"should_exist"` + SizeMiB *int `yaml:"size_mib"` + StartMiB *int `yaml:"start_mib"` + TypeGUID *string `yaml:"type_guid"` + WipePartitionEntry *bool `yaml:"wipe_partition_entry"` +} + +type Passwd struct { + Groups []PasswdGroup `yaml:"groups"` + Users []PasswdUser `yaml:"users"` +} + +type PasswdGroup struct { + Gid *int `yaml:"gid"` + Name string `yaml:"name"` + PasswordHash *string `yaml:"password_hash"` + System *bool `yaml:"system"` +} + +type PasswdUser struct { + Gecos *string `yaml:"gecos"` + Groups []Group `yaml:"groups"` + HomeDir *string `yaml:"home_dir"` + Name string `yaml:"name"` + NoCreateHome *bool `yaml:"no_create_home"` + NoLogInit *bool `yaml:"no_log_init"` + NoUserGroup *bool `yaml:"no_user_group"` + PasswordHash *string `yaml:"password_hash"` + PrimaryGroup *string `yaml:"primary_group"` + SSHAuthorizedKeys []SSHAuthorizedKey `yaml:"ssh_authorized_keys"` + Shell *string `yaml:"shell"` + System *bool `yaml:"system"` + UID *int `yaml:"uid"` +} + +type Raid struct { + Devices []Device `yaml:"devices"` + Level string `yaml:"level"` + Name string `yaml:"name"` + Options []RaidOption `yaml:"options"` + Spares *int `yaml:"spares"` +} + +type RaidOption string + +type SSHAuthorizedKey string + +type Security struct { + TLS TLS `yaml:"tls"` +} + +type Storage struct { + Directories []Directory `yaml:"directories"` + Disks []Disk `yaml:"disks"` + Files []File `yaml:"files"` + Filesystems []Filesystem `yaml:"filesystems"` + Links []Link `yaml:"links"` + Raid []Raid `yaml:"raid"` +} + +type Systemd struct { + Units []Unit `yaml:"units"` +} + +type TLS struct { + CertificateAuthorities []CaReference `yaml:"certificate_authorities"` +} + +type Timeouts struct { + HTTPResponseHeaders *int `yaml:"http_response_headers"` + HTTPTotal *int `yaml:"http_total"` +} + +type Unit struct { + Contents *string `yaml:"contents"` + Dropins []Dropin `yaml:"dropins"` + Enabled *bool `yaml:"enabled"` + Mask *bool `yaml:"mask"` + Name string `yaml:"name"` +} + +type Verification struct { + Hash *string `yaml:"hash"` +} diff --git a/base/v0_2_exp/translate.go b/base/v0_2_exp/translate.go new file mode 100644 index 000000000..e5127794f --- /dev/null +++ b/base/v0_2_exp/translate.go @@ -0,0 +1,101 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_2_exp + +import ( + "net/url" + + "github.com/coreos/fcct/translate" + + "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/coreos/vcontext/path" + "github.com/vincent-petithory/dataurl" +) + +// ToIgn3_0 translates the config to an Ignition config. It also returns the set of translations +// it did so paths in the resultant config can be tracked back to their source in the source config. +func (c Config) ToIgn3_0() (types.Config, translate.TranslationSet, error) { + ret := types.Config{} + tr := translate.NewTranslator("yaml", "json") + tr.AddCustomTranslator(translateIgnition) + tr.AddCustomTranslator(translateFile) + tr.AddCustomTranslator(translateDirectory) + tr.AddCustomTranslator(translateLink) + translations := tr.Translate(&c, &ret) + return ret, translations, nil +} + +func translateIgnition(from Ignition) (to types.Ignition, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + to.Version = types.MaxVersion.String() + tm = tr.Translate(&from.Config, &to.Config).Prefix("config") + tm.MergeP("security", tr.Translate(&from.Security, &to.Security)) + tm.MergeP("timeouts", tr.Translate(&from.Timeouts, &to.Timeouts)) + return +} + +func translateFile(from File) (to types.File, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tr.AddCustomTranslator(translateFileContents) + tm = tr.Translate(&from.Group, &to.Group).Prefix("group") + tm.MergeP("user", tr.Translate(&from.User, &to.User)) + tm.MergeP("append", tr.Translate(&from.Append, &to.Append)) + tm.MergeP("contents", tr.Translate(&from.Contents, &to.Contents)) + to.Overwrite = from.Overwrite + to.Path = from.Path + to.Mode = from.Mode + tm.AddIdentity("overwrite", "path", "mode") + return +} + +func translateFileContents(from FileContents) (to types.FileContents, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Verification, &to.Verification).Prefix("verification") + to.Source = from.Source + to.Compression = from.Compression + tm.AddIdentity("source", "compression") + if from.Inline != nil { + src := (&url.URL{ + Scheme: "data", + Opaque: "," + dataurl.EscapeString(*from.Inline), + }).String() + to.Source = &src + tm.AddTranslation(path.New("yaml", "inline"), path.New("json", "source")) + } + return +} + +func translateDirectory(from Directory) (to types.Directory, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Group, &to.Group).Prefix("group") + tm.MergeP("user", tr.Translate(&from.User, &to.User)) + to.Overwrite = from.Overwrite + to.Path = from.Path + to.Mode = from.Mode + tm.AddIdentity("overwrite", "path", "mode") + return +} + +func translateLink(from Link) (to types.Link, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Group, &to.Group).Prefix("group") + tm.MergeP("user", tr.Translate(&from.User, &to.User)) + to.Target = from.Target + to.Hard = from.Hard + to.Overwrite = from.Overwrite + to.Path = from.Path + tm.AddIdentity("target", "hard", "overwrite", "path") + return +} diff --git a/base/v0_2_exp/translate_test.go b/base/v0_2_exp/translate_test.go new file mode 100644 index 000000000..10de79dc4 --- /dev/null +++ b/base/v0_2_exp/translate_test.go @@ -0,0 +1,324 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_2_exp + +import ( + "reflect" + "testing" + + "github.com/coreos/fcct/translate" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/coreos/vcontext/path" +) + +// Most of this is covered by the Ignition translator generic tests, so just test the custom bits + +// verifyTranslations ensures all the translations are identity, unless they match a listed one +// it returns the offending translation if there is one +func verifyTranslations(set translate.TranslationSet, exceptions ...translate.Translation) *translate.Translation { + exceptionSet := translate.TranslationSet{ + FromTag: set.FromTag, + ToTag: set.ToTag, + Set: map[string]translate.Translation{}, + } + for _, ex := range exceptions { + exceptionSet.AddTranslation(ex.From, ex.To) + } + for key, translation := range set.Set { + if ex, ok := exceptionSet.Set[key]; ok { + if !reflect.DeepEqual(translation, ex) { + return &ex + } + } else if !reflect.DeepEqual(translation.From.Path, translation.To.Path) { + return &translation + } + } + return nil +} + +// TestTranslateFile tests translating the ct storage.files.[i] entries to ignition storage.files.[i] entires. +func TestTranslateFile(t *testing.T) { + tests := []struct { + in File + out types.File + exceptions []translate.Translation + }{ + { + File{}, + types.File{}, + nil, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + File{ + Path: "/foo", + Group: NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Mode: util.IntToPtr(420), + Append: []FileContents{ + { + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + { + Inline: util.StrToPtr("hello"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + Overwrite: util.BoolToPtr(true), + Contents: FileContents{ + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + types.File{ + Node: types.Node{ + Path: "/foo", + Group: types.NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: types.NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + }, + FileEmbedded1: types.FileEmbedded1{ + Mode: util.IntToPtr(420), + Append: []types.FileContents{ + { + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: types.Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + { + Source: util.StrToPtr("data:,hello"), + Compression: util.StrToPtr("gzip"), + Verification: types.Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + Contents: types.FileContents{ + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: types.Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + }, + }, + []translate.Translation{ + { + From: path.New("yaml", "append", 1, "inline"), + To: path.New("json", "append", 1, "source"), + }, + }, + }, + } + + for i, test := range tests { + actual, translations := translateFile(test.in) + + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + + if errT := verifyTranslations(translations, test.exceptions...); errT != nil { + t.Errorf("#%d: bad translation: %v", i, *errT) + } + } +} + +// TestTranslateDirectory tests translating the ct storage.directories.[i] entries to ignition storage.directories.[i] entires. +func TestTranslateDirectory(t *testing.T) { + tests := []struct { + in Directory + out types.Directory + }{ + { + Directory{}, + types.Directory{}, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + Directory{ + Path: "/foo", + Group: NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Mode: util.IntToPtr(420), + Overwrite: util.BoolToPtr(true), + }, + types.Directory{ + Node: types.Node{ + Path: "/foo", + Group: types.NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: types.NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + }, + DirectoryEmbedded1: types.DirectoryEmbedded1{ + Mode: util.IntToPtr(420), + }, + }, + }, + } + + for i, test := range tests { + actual, _ := translateDirectory(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +// TestTranslateLink tests translating the ct storage.links.[i] entries to ignition storage.links.[i] entires. +func TestTranslateLink(t *testing.T) { + tests := []struct { + in Link + out types.Link + }{ + { + Link{}, + types.Link{}, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + Link{ + Path: "/foo", + Group: NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + Target: "/bar", + Hard: util.BoolToPtr(false), + }, + types.Link{ + Node: types.Node{ + Path: "/foo", + Group: types.NodeGroup{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("foobar"), + }, + User: types.NodeUser{ + ID: util.IntToPtr(1), + Name: util.StrToPtr("bazquux"), + }, + Overwrite: util.BoolToPtr(true), + }, + LinkEmbedded1: types.LinkEmbedded1{ + Target: "/bar", + Hard: util.BoolToPtr(false), + }, + }, + }, + } + + for i, test := range tests { + actual, _ := translateLink(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +// TestTranslateIgnition tests translating the ct config.ignition to the ignition config.ignition section. +// It ensure that the version is set as well. +func TestTranslateIgnition(t *testing.T) { + tests := []struct { + in Ignition + out types.Ignition + }{ + { + Ignition{}, + types.Ignition{ + Version: "3.0.0", + }, + }, + } + for i, test := range tests { + actual, _ := translateIgnition(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + +// TestToIgn3_0 tests the config.ToIgn3_0 function ensuring it will generate a valid config even when empty. Not much else is +// tested since it uses the Ignition translation code which has it's own set of tests. +func TestToIgn3_0(t *testing.T) { + tests := []struct { + in Config + out types.Config + }{ + { + Config{}, + types.Config{ + Ignition: types.Ignition{ + Version: "3.0.0", + }, + }, + }, + } + for i, test := range tests { + actual, _, err := test.in.ToIgn3_0() + if err != nil { + t.Errorf("#%d: got error: %v", i, err) + } + + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} diff --git a/base/v0_2_exp/validate.go b/base/v0_2_exp/validate.go new file mode 100644 index 000000000..d60e41c08 --- /dev/null +++ b/base/v0_2_exp/validate.go @@ -0,0 +1,33 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_2_exp + +import ( + "errors" + + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +var ( + ErrInlineAndSource = errors.New("inline cannot be specified if source is specified") +) + +func (f FileContents) Validate(c path.ContextPath) (r report.Report) { + if f.Inline != nil && f.Source != nil { + r.AddOnError(c.Append("inline"), ErrInlineAndSource) + } + return +} diff --git a/base/v0_2_exp/validate_test.go b/base/v0_2_exp/validate_test.go new file mode 100644 index 000000000..2ccb2996f --- /dev/null +++ b/base/v0_2_exp/validate_test.go @@ -0,0 +1,79 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_2_exp + +import ( + "reflect" + "testing" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +// TestValidateFileContents tests that multiple sources (i.e. urls and inline) are not allowed but zero or one sources are +func TestValidateFileContents(t *testing.T) { + tests := []struct { + in FileContents + out error + }{ + {}, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + FileContents{ + Source: util.StrToPtr("http://example/com"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + nil, + }, + { + FileContents{ + Inline: util.StrToPtr("hello"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + nil, + }, + { + FileContents{ + Source: util.StrToPtr("data:,hello"), + Inline: util.StrToPtr("hello"), + Compression: util.StrToPtr("gzip"), + Verification: Verification{ + Hash: util.StrToPtr("this isn't validated"), + }, + }, + ErrInlineAndSource, + }, + } + + for i, test := range tests { + actual := test.in.Validate(path.New("yaml")) + expected := report.Report{} + // hardcode inline for now since that's the only place errors occur. Move into the + // test struct once there's more than one place + expected.AddOnError(path.New("yaml", "inline"), test.out) + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("#%d: expected %+v got %+v", i, expected, actual) + } + } +} From 6d888f71fb8d2ff140d5e7ab1255e4a8e9a7d761 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 11 Oct 2019 11:10:02 -0700 Subject: [PATCH 056/877] base/v0_2_exp: translate to v3.1.0-experimental --- base/v0_2_exp/translate.go | 4 ++-- base/v0_2_exp/translate_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/v0_2_exp/translate.go b/base/v0_2_exp/translate.go index e5127794f..820e6b2a3 100644 --- a/base/v0_2_exp/translate.go +++ b/base/v0_2_exp/translate.go @@ -19,14 +19,14 @@ import ( "github.com/coreos/fcct/translate" - "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/coreos/ignition/v2/config/v3_1_experimental/types" "github.com/coreos/vcontext/path" "github.com/vincent-petithory/dataurl" ) // ToIgn3_0 translates the config to an Ignition config. It also returns the set of translations // it did so paths in the resultant config can be tracked back to their source in the source config. -func (c Config) ToIgn3_0() (types.Config, translate.TranslationSet, error) { +func (c Config) ToIgn3_1() (types.Config, translate.TranslationSet, error) { ret := types.Config{} tr := translate.NewTranslator("yaml", "json") tr.AddCustomTranslator(translateIgnition) diff --git a/base/v0_2_exp/translate_test.go b/base/v0_2_exp/translate_test.go index 10de79dc4..da8829b55 100644 --- a/base/v0_2_exp/translate_test.go +++ b/base/v0_2_exp/translate_test.go @@ -21,7 +21,7 @@ import ( "github.com/coreos/fcct/translate" "github.com/coreos/ignition/v2/config/util" - "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/coreos/ignition/v2/config/v3_1_experimental/types" "github.com/coreos/vcontext/path" ) @@ -283,7 +283,7 @@ func TestTranslateIgnition(t *testing.T) { { Ignition{}, types.Ignition{ - Version: "3.0.0", + Version: "3.1.0-experimental", }, }, } @@ -295,9 +295,9 @@ func TestTranslateIgnition(t *testing.T) { } } -// TestToIgn3_0 tests the config.ToIgn3_0 function ensuring it will generate a valid config even when empty. Not much else is +// TestToIgn3_1 tests the config.ToIgn3_1 function ensuring it will generate a valid config even when empty. Not much else is // tested since it uses the Ignition translation code which has it's own set of tests. -func TestToIgn3_0(t *testing.T) { +func TestToIgn3_1(t *testing.T) { tests := []struct { in Config out types.Config @@ -306,13 +306,13 @@ func TestToIgn3_0(t *testing.T) { Config{}, types.Config{ Ignition: types.Ignition{ - Version: "3.0.0", + Version: "3.1.0-experimental", }, }, }, } for i, test := range tests { - actual, _, err := test.in.ToIgn3_0() + actual, _, err := test.in.ToIgn3_1() if err != nil { t.Errorf("#%d: got error: %v", i, err) } From ff8ba4822502f306b38aa2b53e023dcc418a01d7 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 11 Oct 2019 11:33:34 -0700 Subject: [PATCH 057/877] config/v1_1_exp: add package Add packge to represent the 1.1.0-experimental spec. --- config/v1_1_exp/fcos.go | 96 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 config/v1_1_exp/fcos.go diff --git a/config/v1_1_exp/fcos.go b/config/v1_1_exp/fcos.go new file mode 100644 index 000000000..4d111c029 --- /dev/null +++ b/config/v1_1_exp/fcos.go @@ -0,0 +1,96 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v1_1_exp + +import ( + "errors" + "reflect" + + base_0_2 "github.com/coreos/fcct/base/v0_2_exp" + "github.com/coreos/fcct/config/common" + fcos_0_1 "github.com/coreos/fcct/distro/fcos/v0_1" + "github.com/coreos/fcct/translate" + + "github.com/coreos/ignition/v2/config/v3_1_experimental/types" + ignvalidate "github.com/coreos/ignition/v2/config/validate" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" + "github.com/coreos/vcontext/validate" +) + +var ( + ErrInvalidConfig = errors.New("config generated was invalid") +) + +type Config struct { + common.Common `yaml:",inline"` + base_0_2.Config `yaml:",inline"` + fcos_0_1.Fcos `yaml:",inline"` +} + +func (c Config) Translate() (types.Config, translate.TranslationSet, error) { + cfg, baseTranslations, err := c.Config.ToIgn3_1() + if err != nil { + return types.Config{}, translate.TranslationSet{}, err + } + + finalcfg, distroTranslations, err := c.Fcos.ToIgn3_1(cfg) + if err != nil { + return types.Config{}, translate.TranslationSet{}, err + } + + baseTranslations.Merge(distroTranslations) + + return finalcfg, baseTranslations, nil +} + +// TranslateBytes translates from a v1.0 fcc to a v3.0.0 Ignition config. It returns a report of any errors or +// warnings in the source and resultant config. If the report has fatal errors or it encounters other problems +// translating, an error is returned. +func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, report.Report, error) { + cfg := Config{} + + contextTree, err := common.Unmarshal(input, &cfg, options.Strict) + if err != nil { + return nil, report.Report{}, err + } + + r := validate.Validate(cfg, "yaml") + unusedKeyCheck := func(v reflect.Value, c path.ContextPath) report.Report { + return ignvalidate.ValidateUnusedKeys(v, c, contextTree) + } + r.Merge(validate.ValidateCustom(cfg, "yaml", unusedKeyCheck)) + r.Correlate(contextTree) + if r.IsFatal() { + return nil, r, ErrInvalidConfig + } + + final, translations, err := cfg.Translate() + if err != nil { + return nil, r, err + } + + second := validate.Validate(final, "json") + common.TranslateReportPaths(&second, translations) + second.Correlate(contextTree) + r.Merge(second) + + if r.IsFatal() { + return nil, r, ErrInvalidConfig + } + + outbytes, err := common.Marshal(final, options.Pretty) + return outbytes, r, err +} From 32fe60f1c8063774cd7a7b1f43b0afcad1b3586d Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 11 Oct 2019 11:34:13 -0700 Subject: [PATCH 058/877] config,distro: Start using v1.1.0-experimental Attach the translator for v1.1.0-experimental and allow distro/fcos to also emit Ignition v3.1.0-experimental configs. --- config/config.go | 4 +++- distro/fcos/v0_1/translate.go | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index 1dfb71c42..a1170f6c8 100644 --- a/config/config.go +++ b/config/config.go @@ -20,6 +20,7 @@ import ( "github.com/coreos/fcct/config/common" "github.com/coreos/fcct/config/v1_0" + "github.com/coreos/fcct/config/v1_1_exp" "github.com/coreos/go-semver/semver" "github.com/coreos/vcontext/report" @@ -31,7 +32,8 @@ var ( ErrInvalidVersion = errors.New("Error parsing version. Version must be a valid semver") registry = map[string]translator{ - "fcos+1.0.0": v1_0.TranslateBytes, + "fcos+1.0.0": v1_0.TranslateBytes, + "fcos+1.1.0-experimental": v1_1_exp.TranslateBytes, } ) diff --git a/distro/fcos/v0_1/translate.go b/distro/fcos/v0_1/translate.go index 17918bc0d..116ca3f46 100644 --- a/distro/fcos/v0_1/translate.go +++ b/distro/fcos/v0_1/translate.go @@ -17,10 +17,15 @@ package fcos_0_1 import ( "github.com/coreos/fcct/translate" - "github.com/coreos/ignition/v2/config/v3_0/types" + types3_0 "github.com/coreos/ignition/v2/config/v3_0/types" + types3_1 "github.com/coreos/ignition/v2/config/v3_1_experimental/types" ) // ToIgn3_0 takes a config and merges in the distro specific bits. -func (f Fcos) ToIgn3_0(in types.Config) (types.Config, translate.TranslationSet, error) { +func (f Fcos) ToIgn3_0(in types3_0.Config) (types3_0.Config, translate.TranslationSet, error) { + return in, translate.TranslationSet{}, nil +} + +func (f Fcos) ToIgn3_1(in types3_1.Config) (types3_1.Config, translate.TranslationSet, error) { return in, translate.TranslationSet{}, nil } From 43c0e57490dcbb89374535a1887718d659597836 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 14 Oct 2019 15:10:36 -0700 Subject: [PATCH 059/877] docs: add docs for v1.1.0-experimental Add docs as a copy of v1.0.0, with the version changed and a warning about it being experimental. No changes to the contents yet, that will happen when the fields are added to FCCT. --- docs/configuration-v1_1-exp.md | 128 +++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 docs/configuration-v1_1-exp.md diff --git a/docs/configuration-v1_1-exp.md b/docs/configuration-v1_1-exp.md new file mode 100644 index 000000000..05ac688e0 --- /dev/null +++ b/docs/configuration-v1_1-exp.md @@ -0,0 +1,128 @@ +# Configuration Specification v1.1.0-experimental # + +**Note: This configuration is experimental and has not been stablized. It is subject to change without warning or announcement** + +The Fedora CoreOS configuration is a YAML document conforming to the following specification, with **_italicized_** entries being optional: + +* **variant** (string): used to differentiate configs for different operating systems. Must be `fcos` for FCCT. +* **version** (string): the semantic version of the spec for this document. This document is for version `1.0.0` and generates Ignition configs with version `3.0.0`. +* **ignition** (object): metadata about the configuration itself. + * **_config_** (objects): options related to the configuration. + * **_merge_** (list of objects): a list of the configs to be merged to the current config. + * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the config. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_replace_** (object): the config that will replace the current. + * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the config. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_timeouts_** (object): options relating to `http` timeouts when fetching files over `http` or `https`. + * **_http_response_headers_** (integer) the time to wait (in seconds) for the server's response headers (but not the body) after making a request. 0 indicates no timeout. Default is 10 seconds. + * **_http_total_** (integer) the time limit (in seconds) for the operation (connection, request, and response), including retries. 0 indicates no timeout. Default is 0. + * **_security_** (object): options relating to network security. + * **_tls_** (object): options relating to TLS when fetching resources over `https`. + * **_certificate_authorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`. + * **source** (string): the URL of the certificate (in PEM format). Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the certificate. + * **_hash_** (string): the hash of the certificate, in the form `-` where type is sha512. +* **_storage_** (object): describes the desired state of the system's storage devices. + * **_disks_** (list of objects): the list of disks to be configured and their options. Every entry must have a unique `device`. + * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. + * **_wipe_table_** (boolean): whether or not the partition tables shall be wiped. When true, the partition tables are erased before any further manipulation. Otherwise, the existing entries are left intact. + * **_partitions_** (list of objects): the list of partitions and their configuration for this particular disk. Every partition must have a unique `number`, or if 0 is specified, a unique `label`. + * **_label_** (string): the PARTLABEL for the partition. + * **_number_** (integer): the partition number, which dictates it's position in the partition table (one-indexed). If zero, use the next available partition slot. + * **_size_mib_** (integer): the size of the partition (in mebibytes). If zero, the partition will be made as large as possible. + * **_start_mib_** (integer): the start of the partition (in mebibytes). If zero, the partition will be positioned at the start of the largest block available. + * **_type_guid_** (string): the GPT [partition type GUID][part-types]. If omitted, the default will be 0FC63DAF-8483-4772-8E79-3D69D8477DE4 (Linux filesystem data). + * **_guid_** (string): the GPT unique partition GUID. + * **_wipe_partition_entry_** (boolean) if true, Ignition will clobber an existing partition if it does not match the config. If false (default), Ignition will fail instead. + * **_should_exist_** (boolean) whether or not the partition with the specified `number` should exist. If omitted, it defaults to true. If false Ignition will either delete the specified partition or fail, depending on `wipePartitionEntry`. If false `number` must be specified and non-zero and `label`, `start`, `size`, `guid`, and `typeGuid` must all be omitted. + * **_raid_** (list of objects): the list of RAID arrays to be configured. Every RAID array must have a unique `name`. + * **name** (string): the name to use for the resulting md device. + * **level** (string): the redundancy level of the array (e.g. linear, raid1, raid5, etc.). + * **devices** (list of strings): the list of devices (referenced by their absolute path) in the array. + * **_spares_** (integer): the number of spares (if applicable) in the array. + * **_options_** (list of strings): any additional options to be passed to mdadm. + * **_filesystems_** (list of objects): the list of filesystems to be configured. `path`, `device`, and `format` all need to be specified. Every filesystem must have a unique `device`. + * **path** (string): the mount-point of the filesystem while Ignition is running relative to where the root filesystem will be mounted. This is not necessarily the same as where it should be mounted in the real root, but it is encouraged to make it the same. + * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. + * **format** (string): the filesystem format (ext4, btrfs, xfs, vfat, or swap). + * **_wipe_filesystem_** (boolean): whether or not to wipe the device before filesystem creation, see [the documentation on filesystems](operator-notes.md#filesystem-reuse-semantics) for more information. + * **_label_** (string): the label of the filesystem. + * **_uuid_** (string): the uuid of the filesystem. + * **_options_** (list of strings): any additional options to be passed to the format-specific mkfs utility. + * **_files_** (list of objects): the list of files to be written. Every file, directory and link must have a unique `path`. + * **path** (string): the absolute path to the file. + * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `source` must be specified if `overwrite` is true. Defaults to false. + * **_contents_** (object): options related to the contents of the file. + * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. + * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created. Mutually exclusive with `inline`. + * **_inline_** (string): the contents of the file. Mutually exclusive with `source`. + * **_verification_** (object): options related to the verification of the file contents. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents` + * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. + * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_verification_** (object): options related to the verification of the appended contents. + * **_hash_** (string): the hash of the config, in the form `-` where type is `sha512`. + * **_mode_** (integer): the file's permission mode. If not specified, the permission mode for files defaults to 0644 or the existing file's permissions if `overwrite` is false, `source` is unspecified, and a file already exists at the path. + * **_user_** (object): specifies the file's owner. + * **_id_** (integer): the user ID of the owner. + * **_name_** (string): the user name of the owner. + * **_group_** (object): specifies the group of the owner. + * **_id_** (integer): the group ID of the owner. + * **_name_** (string): the group name of the owner. + * **_directories_** (list of objects): the list of directories to be created. Every file, directory, and link must have a unique `path`. + * **path** (string): the absolute path to the directory. + * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If false and a directory already exists at the path, Ignition will only set its permissions. If false and a non-directory exists at that path, Ignition will fail. Defaults to false. + * **_mode_** (integer): the directory's permission mode. If not specified, the permission mode for directories defaults to 0755 or the mode of an existing directory if `overwrite` is false and a directory already exists at the path. + * **_user_** (object): specifies the directory's owner. + * **_id_** (integer): the user ID of the owner. + * **_name_** (string): the user name of the owner. + * **_group_** (object): specifies the group of the owner. + * **_id_** (integer): the group ID of the owner. + * **_name_** (string): the group name of the owner. + * **_links_** (list of objects): the list of links to be created. Every file, directory, and link must have a unique `path`. + * **path** (string): the absolute path to the link + * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If overwrite is false and a matching link exists at the path, Ignition will only set the owner and group. Defaults to false. + * **_user_** (object): specifies the symbolic link's owner. + * **_id_** (integer): the user ID of the owner. + * **_name_** (string): the user name of the owner. + * **_group_** (object): specifies the group of the owner. + * **_id_** (integer): the group ID of the owner. + * **_name_** (string): the group name of the owner. + * **target** (string): the target path of the link + * **_hard_** (boolean): a symbolic link is created if this is false, a hard one if this is true. +* **_systemd_** (object): describes the desired state of the systemd units. + * **_units_** (list of objects): the list of systemd units. + * **name** (string): the name of the unit. This must be suffixed with a valid unit type (e.g. "thing.service"). Every unit must have a unique `name`. + * **_enabled_** (boolean): whether or not the service shall be enabled. When true, the service is enabled. When false, the service is disabled. When omitted, the service is unmodified. In order for this to have any effect, the unit must have an install section. + * **_mask_** (boolean): whether or not the service shall be masked. When true, the service is masked by symlinking it to `/dev/null`. + * **_contents_** (string): the contents of the unit. + * **_dropins_** (list of objects): the list of drop-ins for the unit. Every drop-in must have a unique `name`. + * **name** (string): the name of the drop-in. This must be suffixed with ".conf". + * **_contents_** (string): the contents of the drop-in. +* **_passwd_** (object): describes the desired additions to the passwd database. + * **_users_** (list of objects): the list of accounts that shall exist. All users must have a unique `name`. + * **name** (string): the username for the account. + * **_password_hash_** (string): the hashed password for the account. + * **_ssh_authorized_keys_** (list of strings): a list of SSH keys to be added as an SSH key fragment at `.ssh/authorized_keys.d/ignition` in the user's home directory. All SSH keys must be unique. + * **_uid_** (integer): the user ID of the account. + * **_gecos_** (string): the GECOS field of the account. + * **_home_dir_** (string): the home directory of the account. + * **_no_create_home_** (boolean): whether or not to create the user's home directory. This only has an effect if the account doesn't exist yet. + * **_primary_group_** (string): the name of the primary group of the account. + * **_groups_** (list of strings): the list of supplementary groups of the account. + * **_no_user_group_** (boolean): whether or not to create a group with the same name as the user. This only has an effect if the account doesn't exist yet. + * **_no_log_init_** (boolean): whether or not to add the user to the lastlog and faillog databases. This only has an effect if the account doesn't exist yet. + * **_shell_** (string): the login shell of the new account. + * **_system_** (bool): whether or not this account should be a system account. This only has an effect if the account doesn't exist yet. + * **_groups_** (list of objects): the list of groups to be added. All groups must have a unique `name`. + * **name** (string): the name of the group. + * **_gid_** (integer): the group ID of the new group. + * **_password_hash_** (string): the hashed password of the new group. + * **_system_** (bool): whether or not the group should be a system group. This only has an effect if the group doesn't exist yet. + +[part-types]: http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs +[rfc2397]: https://tools.ietf.org/html/rfc2397 From e41fd7084bff8a1a04a4e498ef292b4ca1ff2b6c Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 14 Oct 2019 15:35:49 -0700 Subject: [PATCH 060/877] base/v0_2: add new fields from ign spec 3.1.0-exp Add the proxy field which is new in Ignition spec 3.1.0-experimental. --- base/v0_2_exp/schema.go | 7 +++++++ base/v0_2_exp/translate.go | 1 + base/v0_2_exp/translate_test.go | 15 +++++++++++++++ docs/configuration-v1_1-exp.md | 4 ++++ 4 files changed, 27 insertions(+) diff --git a/base/v0_2_exp/schema.go b/base/v0_2_exp/schema.go index 8ed19118a..8c4221189 100644 --- a/base/v0_2_exp/schema.go +++ b/base/v0_2_exp/schema.go @@ -85,6 +85,7 @@ type Group string type Ignition struct { Config IgnitionConfig `yaml:"config"` + Proxy Proxy `yaml:"proxy"` Security Security `yaml:"security"` Timeouts Timeouts `yaml:"timeouts"` } @@ -152,6 +153,12 @@ type PasswdUser struct { UID *int `yaml:"uid"` } +type Proxy struct { + HTTPProxy *string `yaml:"http_proxy"` + HTTPSProxy *string `yaml:"https_proxy"` + NoProxy []string `yaml:"no_proxy"` +} + type Raid struct { Devices []Device `yaml:"devices"` Level string `yaml:"level"` diff --git a/base/v0_2_exp/translate.go b/base/v0_2_exp/translate.go index 820e6b2a3..c5dec906a 100644 --- a/base/v0_2_exp/translate.go +++ b/base/v0_2_exp/translate.go @@ -41,6 +41,7 @@ func translateIgnition(from Ignition) (to types.Ignition, tm translate.Translati tr := translate.NewTranslator("yaml", "json") to.Version = types.MaxVersion.String() tm = tr.Translate(&from.Config, &to.Config).Prefix("config") + tm.MergeP("proxy", tr.Translate(&from.Proxy, &to.Proxy)) tm.MergeP("security", tr.Translate(&from.Security, &to.Security)) tm.MergeP("timeouts", tr.Translate(&from.Timeouts, &to.Timeouts)) return diff --git a/base/v0_2_exp/translate_test.go b/base/v0_2_exp/translate_test.go index da8829b55..a11db772c 100644 --- a/base/v0_2_exp/translate_test.go +++ b/base/v0_2_exp/translate_test.go @@ -286,6 +286,21 @@ func TestTranslateIgnition(t *testing.T) { Version: "3.1.0-experimental", }, }, + { + Ignition{ + Proxy: Proxy{ + HTTPProxy: util.StrToPtr("https://example.com:8080"), + NoProxy: []string{"example.com"}, + }, + }, + types.Ignition{ + Version: "3.1.0-experimental", + Proxy: types.Proxy{ + HTTPProxy: util.StrToPtr("https://example.com:8080"), + NoProxy: []types.NoProxyItem{types.NoProxyItem("example.com")}, + }, + }, + }, } for i, test := range tests { actual, _ := translateIgnition(test.in) diff --git a/docs/configuration-v1_1-exp.md b/docs/configuration-v1_1-exp.md index 05ac688e0..d1406613d 100644 --- a/docs/configuration-v1_1-exp.md +++ b/docs/configuration-v1_1-exp.md @@ -25,6 +25,10 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **source** (string): the URL of the certificate (in PEM format). Supported schemes are `http`, `https`, `s3`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_verification_** (object): options related to the verification of the certificate. * **_hash_** (string): the hash of the certificate, in the form `-` where type is sha512. + * **_proxy_** (object): options relating to setting an `HTTP(S)` proxy when fetching resources. + * **_httpProxy_** (string): will be used as the proxy URL for HTTP requests and HTTPS requests unless overridden by `httpsProxy` or `noProxy`. + * **_httpsProxy_** (string): will be used as the proxy URL for HTTPS requests unless overridden by `noProxy`. + * **_noProxy_** (list of strings): specifies a list of strings to hosts that should be excluded from proxying. Each value is represented by an `IP address prefix (1.2.3.4)`, `an IP address prefix in CIDR notation (1.2.3.4/8)`, `a domain name`, or `a special DNS label (*)`. An IP address prefix and domain name can also include a literal port number `(1.2.3.4:80)`. A domain name matches that name and all subdomains. A domain name with a leading `.` matches subdomains only. For example `foo.com` matches `foo.com` and `bar.foo.com`; `.y.com` matches `x.y.com` but not `y.com`. A single asterisk `(*)` indicates that no proxying should be done. * **_storage_** (object): describes the desired state of the system's storage devices. * **_disks_** (list of objects): the list of disks to be configured and their options. Every entry must have a unique `device`. * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. From f051d9a2eabc075492f9640694b6cb97a6249f4c Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 15 Oct 2019 11:19:22 -0700 Subject: [PATCH 061/877] base/v0_2: add field to create mount units for fs Add an option to storage.filesystems to create a mount unit. This doesn't work yet, just creates the field and adds a unity translator that ignores the field. Also get rid of the type FilesystemOption. It only exists since the types were copied over from Ignition and were generated by schematyper. --- base/v0_2_exp/schema.go | 15 ++++++------ base/v0_2_exp/translate.go | 14 +++++++++++ base/v0_2_exp/translate_test.go | 43 +++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/base/v0_2_exp/schema.go b/base/v0_2_exp/schema.go index 8c4221189..efe9bc51d 100644 --- a/base/v0_2_exp/schema.go +++ b/base/v0_2_exp/schema.go @@ -70,13 +70,14 @@ type FileContents struct { } type Filesystem struct { - Device string `yaml:"device"` - Format *string `yaml:"format"` - Label *string `yaml:"label"` - Options []FilesystemOption `yaml:"options"` - Path *string `yaml:"path"` - UUID *string `yaml:"uuid"` - WipeFilesystem *bool `yaml:"wipe_filesystem"` + Device string `yaml:"device"` + Format *string `yaml:"format"` + Label *string `yaml:"label"` + Options []string `yaml:"options"` + Path *string `yaml:"path"` + UUID *string `yaml:"uuid"` + WipeFilesystem *bool `yaml:"wipe_filesystem"` + WithMountUnit *bool `yaml:"with_mount_unit"` } type FilesystemOption string diff --git a/base/v0_2_exp/translate.go b/base/v0_2_exp/translate.go index c5dec906a..c85e66ceb 100644 --- a/base/v0_2_exp/translate.go +++ b/base/v0_2_exp/translate.go @@ -33,6 +33,7 @@ func (c Config) ToIgn3_1() (types.Config, translate.TranslationSet, error) { tr.AddCustomTranslator(translateFile) tr.AddCustomTranslator(translateDirectory) tr.AddCustomTranslator(translateLink) + tr.AddCustomTranslator(translateFilesystem) translations := tr.Translate(&c, &ret) return ret, translations, nil } @@ -100,3 +101,16 @@ func translateLink(from Link) (to types.Link, tm translate.TranslationSet) { tm.AddIdentity("target", "hard", "overwrite", "path") return } + +func translateFilesystem(from Filesystem) (to types.Filesystem, tm translate.TranslationSet) { + tr := translate.NewTranslator("yaml", "json") + tm = tr.Translate(&from.Options, &to.Options).Prefix("options") + to.Device = from.Device + to.Label = from.Label + to.Format = from.Format + to.Path = from.Path + to.UUID = from.UUID + to.WipeFilesystem = from.WipeFilesystem + tm.AddIdentity("device", "format", "label", "path", "uuid", "wipeFilesystem") + return +} diff --git a/base/v0_2_exp/translate_test.go b/base/v0_2_exp/translate_test.go index a11db772c..df22ef710 100644 --- a/base/v0_2_exp/translate_test.go +++ b/base/v0_2_exp/translate_test.go @@ -273,6 +273,49 @@ func TestTranslateLink(t *testing.T) { } } +// TestTranslateFilesystem tests translating the ct storage.filesystems.[i] entries to ignition storage.filesystems.[i] entries. +func TestTranslateFilesystem(t *testing.T) { + tests := []struct { + in Filesystem + out types.Filesystem + }{ + { + Filesystem{}, + types.Filesystem{}, + }, + { + // contains invalid (by the validator's definition) combinations of fields, + // but the translator doesn't care and we can check they all get translated at once + Filesystem{ + Device: "/foo", + Format: util.StrToPtr("/bar"), + Label: util.StrToPtr("/baz"), + Options: []string{"foo", "foo", "bar"}, + Path: util.StrToPtr("/quux"), + UUID: util.StrToPtr("1234"), + WipeFilesystem: util.BoolToPtr(true), + WithMountUnit: util.BoolToPtr(true), + }, + types.Filesystem{ + Device: "/foo", + Format: util.StrToPtr("/bar"), + Label: util.StrToPtr("/baz"), + Options: []types.FilesystemOption{"foo", "foo", "bar"}, + Path: util.StrToPtr("/quux"), + UUID: util.StrToPtr("1234"), + WipeFilesystem: util.BoolToPtr(true), + }, + }, + } + + for i, test := range tests { + actual, _ := translateFilesystem(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("#%d: expected %+v got %+v", i, test.out, actual) + } + } +} + // TestTranslateIgnition tests translating the ct config.ignition to the ignition config.ignition section. // It ensure that the version is set as well. func TestTranslateIgnition(t *testing.T) { From 6c94158dd1cd384de66982e010e62bbb30469b3c Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Tue, 15 Oct 2019 16:05:25 -0700 Subject: [PATCH 062/877] translate: minor refactor, add tests, fix bug Do a small bit of refactoring resulting from debugging. Add a String() method to TranslationSet's to aid debugging. Fix a bug where TranslationSet's would have path.ContextPath's with aliased arrays. --- translate/translate.go | 38 ++++++++++++++++++++++++------------- translate/translate_test.go | 32 +++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/translate/translate.go b/translate/translate.go index d94476fc5..8ca3f8690 100644 --- a/translate/translate.go +++ b/translate/translate.go @@ -211,8 +211,27 @@ type TranslationSet struct { Set map[string]Translation } +func NewTranslationSet(fromTag, toTag string) TranslationSet { + return TranslationSet{ + FromTag: fromTag, + ToTag: toTag, + Set: map[string]Translation{}, + } +} + +func (ts TranslationSet) String() string { + str := fmt.Sprintf("from: %v\nto: %v\n", ts.FromTag, ts.ToTag) + for k, v := range ts.Set { + str += fmt.Sprintf("%s: %v -> %v\n", k, v.From.String(), v.To.String()) + } + return str +} + // AddTranslation adds a translation to the set func (ts TranslationSet) AddTranslation(from, to path.ContextPath) { + // create copies of the paths so if someone else changes from.Path the added translation does not change. + from = from.Copy() + to = to.Copy() translation := Translation{ From: from, To: to, @@ -233,30 +252,23 @@ func (ts TranslationSet) AddIdentity(paths ...string) { // Merge adds all the entries to the set. It mutates the Set in place. func (ts TranslationSet) Merge(from TranslationSet) { for _, t := range from.Set { - ts.Set[t.From.String()] = t + ts.AddTranslation(t.From, t.To) } } // MergeP is like Merge, but first it calls Prefix on the set being merged in. func (ts TranslationSet) MergeP(prefix interface{}, from TranslationSet) { from = from.Prefix(prefix) - for _, t := range from.Set { - ts.Set[t.To.String()] = t - } + ts.Merge(from) } // Prefix returns a TranslationSet with all translation paths prefixed by prefix. func (ts TranslationSet) Prefix(prefix interface{}) TranslationSet { - ret := TranslationSet{ - FromTag: ts.FromTag, - ToTag: ts.ToTag, - Set: map[string]Translation{}, - } - p := []interface{}{prefix} + ret := NewTranslationSet(ts.FromTag, ts.ToTag) + from := path.New(ts.FromTag, prefix) + to := path.New(ts.ToTag, prefix) for _, tr := range ts.Set { - tr.From.Path = append(p, tr.From.Path...) - tr.To.Path = append(p, tr.To.Path...) - ret.AddTranslation(tr.From, tr.To) + ret.AddTranslation(from.Append(tr.From.Path...), to.Append(tr.From.Path...)) } return ret } diff --git a/translate/translate_test.go b/translate/translate_test.go index a6396eadf..eb40c854f 100644 --- a/translate/translate_test.go +++ b/translate/translate_test.go @@ -274,3 +274,35 @@ func TestCustomTranslatorList(t *testing.T) { assert.Equal(t, got, expected, "bad translation") assert.Equal(t, ts, exTrans, "bad translation") } + +func TestAddIdentity(t *testing.T) { + ts := NewTranslationSet("1", "2") + ts.AddIdentity("foo", "bar") + expectedFoo := Translation{ + From: path.New("1", "foo"), + To: path.New("2", "foo"), + } + expectedBar := Translation{ + From: path.New("1", "bar"), + To: path.New("2", "bar"), + } + expectedFoo2 := Translation{ + From: path.New("1", "pre", "foo"), + To: path.New("2", "pre", "foo"), + } + expectedBar2 := Translation{ + From: path.New("1", "pre", "bar"), + To: path.New("2", "pre", "bar"), + } + ts2 := NewTranslationSet("1", "2") + ts2.MergeP("pre", ts) + ts3 := NewTranslationSet("1", "2") + ts3.Merge(ts.Prefix("pre")) + + assert.Equal(t, ts.Set["$.foo"], expectedFoo, "foo not added correctly") + assert.Equal(t, ts.Set["$.bar"], expectedBar, "bar not added correctly") + assert.Equal(t, ts2.Set["$.pre.foo"], expectedFoo2, "foo not added correctly") + assert.Equal(t, ts3.Set["$.pre.bar"], expectedBar2, "bar not added correctly") + assert.Equal(t, ts3.Set["$.pre.foo"], expectedFoo2, "foo not added correctly") + assert.Equal(t, ts2.Set["$.pre.bar"], expectedBar2, "bar not added correctly") +} From 0928bd7507584d4c2b8aa685c5a504fc7e6732ab Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 16 Oct 2019 10:45:23 -0700 Subject: [PATCH 063/877] translate: split TranslationSet to own file Split TranslationSet code into its own file, having everything in one file was getting messy. --- translate/set.go | 101 +++++++++++++++++++++++++++++++++++++++++ translate/translate.go | 80 -------------------------------- 2 files changed, 101 insertions(+), 80 deletions(-) create mode 100644 translate/set.go diff --git a/translate/set.go b/translate/set.go new file mode 100644 index 000000000..8ca50469e --- /dev/null +++ b/translate/set.go @@ -0,0 +1,101 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translate + +import ( + "fmt" + + "github.com/coreos/vcontext/path" +) + +// Translation represents how a path changes when translating. If something at $yaml.storage.filesystems.4 +// generates content at $json.systemd.units.3 a translation can represent that. This allows validation errors +// in Ignition structs to be tracked back to their source in the yaml. +type Translation struct { + From path.ContextPath + To path.ContextPath +} + +// TranslationSet represents all of the translations that occurred. They're stored in a map from a string representation +// of the destination path to the translation struct. The map is purely an optimization to allow fast lookups. Ideally the +// map would just be from the destination path.ContextPath to the source path.ContextPath, but ContextPath contains a slice +// which are not comparable and thus cannot be used as keys in maps. +type TranslationSet struct { + FromTag string + ToTag string + Set map[string]Translation +} + +func NewTranslationSet(fromTag, toTag string) TranslationSet { + return TranslationSet{ + FromTag: fromTag, + ToTag: toTag, + Set: map[string]Translation{}, + } +} + +func (ts TranslationSet) String() string { + str := fmt.Sprintf("from: %v\nto: %v\n", ts.FromTag, ts.ToTag) + for k, v := range ts.Set { + str += fmt.Sprintf("%s: %v -> %v\n", k, v.From.String(), v.To.String()) + } + return str +} + +// AddTranslation adds a translation to the set +func (ts TranslationSet) AddTranslation(from, to path.ContextPath) { + // create copies of the paths so if someone else changes from.Path the added translation does not change. + from = from.Copy() + to = to.Copy() + translation := Translation{ + From: from, + To: to, + } + toString := translation.To.String() + ts.Set[toString] = translation +} + +// Shortcut for AddTranslation for identity translations +func (ts TranslationSet) AddIdentity(paths ...string) { + for _, p := range paths { + from := path.New(ts.FromTag, p) + to := path.New(ts.ToTag, p) + ts.AddTranslation(from, to) + } +} + +// Merge adds all the entries to the set. It mutates the Set in place. +func (ts TranslationSet) Merge(from TranslationSet) { + for _, t := range from.Set { + ts.AddTranslation(t.From, t.To) + } +} + +// MergeP is like Merge, but first it calls Prefix on the set being merged in. +func (ts TranslationSet) MergeP(prefix interface{}, from TranslationSet) { + from = from.Prefix(prefix) + ts.Merge(from) +} + +// Prefix returns a TranslationSet with all translation paths prefixed by prefix. +func (ts TranslationSet) Prefix(prefix interface{}) TranslationSet { + ret := NewTranslationSet(ts.FromTag, ts.ToTag) + from := path.New(ts.FromTag, prefix) + to := path.New(ts.ToTag, prefix) + for _, tr := range ts.Set { + ret.AddTranslation(from.Append(tr.From.Path...), to.Append(tr.From.Path...)) + } + return ret +} diff --git a/translate/translate.go b/translate/translate.go index 8ca3f8690..3662f8e96 100644 --- a/translate/translate.go +++ b/translate/translate.go @@ -193,86 +193,6 @@ type Translator interface { Translate(from, to interface{}) TranslationSet } -// Translation represents how a path changes when translating. If something at $yaml.storage.filesystems.4 -// generates content at $json.systemd.units.3 a translation can represent that. This allows validation errors -// in Ignition structs to be tracked back to their source in the yaml. -type Translation struct { - From path.ContextPath - To path.ContextPath -} - -// TranslationSet represents all of the translations that occurred. They're stored in a map from a string representation -// of the destination path to the translation struct. The map is purely an optimization to allow fast lookups. Ideally the -// map would just be from the destination path.ContextPath to the source path.ContextPath, but ContextPath contains a slice -// which are not comparable and thus cannot be used as keys in maps. -type TranslationSet struct { - FromTag string - ToTag string - Set map[string]Translation -} - -func NewTranslationSet(fromTag, toTag string) TranslationSet { - return TranslationSet{ - FromTag: fromTag, - ToTag: toTag, - Set: map[string]Translation{}, - } -} - -func (ts TranslationSet) String() string { - str := fmt.Sprintf("from: %v\nto: %v\n", ts.FromTag, ts.ToTag) - for k, v := range ts.Set { - str += fmt.Sprintf("%s: %v -> %v\n", k, v.From.String(), v.To.String()) - } - return str -} - -// AddTranslation adds a translation to the set -func (ts TranslationSet) AddTranslation(from, to path.ContextPath) { - // create copies of the paths so if someone else changes from.Path the added translation does not change. - from = from.Copy() - to = to.Copy() - translation := Translation{ - From: from, - To: to, - } - toString := translation.To.String() - ts.Set[toString] = translation -} - -// Shortcut for AddTranslation for identity translations -func (ts TranslationSet) AddIdentity(paths ...string) { - for _, p := range paths { - from := path.New(ts.FromTag, p) - to := path.New(ts.ToTag, p) - ts.AddTranslation(from, to) - } -} - -// Merge adds all the entries to the set. It mutates the Set in place. -func (ts TranslationSet) Merge(from TranslationSet) { - for _, t := range from.Set { - ts.AddTranslation(t.From, t.To) - } -} - -// MergeP is like Merge, but first it calls Prefix on the set being merged in. -func (ts TranslationSet) MergeP(prefix interface{}, from TranslationSet) { - from = from.Prefix(prefix) - ts.Merge(from) -} - -// Prefix returns a TranslationSet with all translation paths prefixed by prefix. -func (ts TranslationSet) Prefix(prefix interface{}) TranslationSet { - ret := NewTranslationSet(ts.FromTag, ts.ToTag) - from := path.New(ts.FromTag, prefix) - to := path.New(ts.ToTag, prefix) - for _, tr := range ts.Set { - ret.AddTranslation(from.Append(tr.From.Path...), to.Append(tr.From.Path...)) - } - return ret -} - // NewTranslator creates a new Translator for translating from types with fromTag struct tags (e.g. "yaml") // to types with toTag struct tages (e.g. "json"). These tags are used when determining paths when generating // the TranslationSet returned by Translator.Translate() From b2eebed5a9eb7bae0df21d35814e2457f1f44c2b Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 16 Oct 2019 10:57:04 -0700 Subject: [PATCH 064/877] translate: split helper fns to their own file --- translate/translate.go | 11 ----------- translate/util.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 translate/util.go diff --git a/translate/translate.go b/translate/translate.go index 3662f8e96..3cbfbf459 100644 --- a/translate/translate.go +++ b/translate/translate.go @@ -17,7 +17,6 @@ package translate import ( "fmt" "reflect" - "strings" "github.com/coreos/ignition/v2/config/util" "github.com/coreos/vcontext/path" @@ -96,16 +95,6 @@ func couldBeValidTranslator(t reflect.Type) bool { return true } -// fieldName returns the name uses when (un)marshalling a field. t should be a reflect.Value of a struct, -// index is the field index, and tag is the struct tag used when (un)marshalling (e.g. "json" or "yaml") -func fieldName(t reflect.Value, index int, tag string) string { - f := t.Type().Field(index) - if tag == "" { - return f.Name - } - return strings.Split(f.Tag.Get(tag), ",")[0] -} - // translate from one type to another, but deep copy all data // precondition: vFrom and vTo are the same type as defined by translatable() // precondition: vTo is addressable and settable diff --git a/translate/util.go b/translate/util.go new file mode 100644 index 000000000..8412b4762 --- /dev/null +++ b/translate/util.go @@ -0,0 +1,30 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translate + +import ( + "reflect" + "strings" +) + +// fieldName returns the name uses when (un)marshalling a field. t should be a reflect.Value of a struct, +// index is the field index, and tag is the struct tag used when (un)marshalling (e.g. "json" or "yaml") +func fieldName(t reflect.Value, index int, tag string) string { + f := t.Type().Field(index) + if tag == "" { + return f.Name + } + return strings.Split(f.Tag.Get(tag), ",")[0] +} From 4d6b5461eeb03dd12d57844574f7867926c42d1d Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 16 Oct 2019 13:52:30 -0700 Subject: [PATCH 065/877] translate: allow translations from a common source Add support for generating translations to a generated path from a single path. This is useful for when FCCT generates an chunk of a config all from one part of the FCC. --- translate/set.go | 12 ++++++++++++ translate/util.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/translate/set.go b/translate/set.go index 8ca50469e..6140fbf1b 100644 --- a/translate/set.go +++ b/translate/set.go @@ -16,6 +16,7 @@ package translate import ( "fmt" + "reflect" "github.com/coreos/vcontext/path" ) @@ -76,6 +77,17 @@ func (ts TranslationSet) AddIdentity(paths ...string) { } } +// AddFromCommonSource adds translations for all of the paths in to from a single common path. This is useful +// if one part of a config generates a large struct and all of the large struct should map to one path in the +// config being translated. +func (ts TranslationSet) AddFromCommonSource(common path.ContextPath, toPrefix path.ContextPath, to interface{}) { + v := reflect.ValueOf(to) + vPaths := prefixPaths(getAllPaths(v, ts.ToTag), toPrefix.Path...) + for _, path := range vPaths { + ts.AddTranslation(common, path) + } +} + // Merge adds all the entries to the set. It mutates the Set in place. func (ts TranslationSet) Merge(from TranslationSet) { for _, t := range from.Set { diff --git a/translate/util.go b/translate/util.go index 8412b4762..6f2b6dd50 100644 --- a/translate/util.go +++ b/translate/util.go @@ -17,6 +17,9 @@ package translate import ( "reflect" "strings" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/vcontext/path" ) // fieldName returns the name uses when (un)marshalling a field. t should be a reflect.Value of a struct, @@ -28,3 +31,50 @@ func fieldName(t reflect.Value, index int, tag string) string { } return strings.Split(f.Tag.Get(tag), ",")[0] } + +func prefixPath(p path.ContextPath, prefix ...interface{}) path.ContextPath { + return path.New(p.Tag, prefix...).Append(p.Path...) +} + +func prefixPaths(ps []path.ContextPath, prefix ...interface{}) []path.ContextPath { + ret := []path.ContextPath{} + for _, p := range ps { + ret = append(ret, prefixPath(p, prefix...)) + } + return ret +} + +func getAllPaths(v reflect.Value, tag string) []path.ContextPath { + k := v.Kind() + t := v.Type() + switch { + case util.IsPrimitive(k): + return nil + case k == reflect.Ptr: + if v.IsNil() { + return nil + } + return getAllPaths(v.Elem(), tag) + case k == reflect.Slice: + ret := []path.ContextPath{} + for i := 0; i < v.Len(); i++ { + ret = append(ret, prefixPaths(getAllPaths(v.Index(i), tag), i)...) + } + return ret + case k == reflect.Struct: + ret := []path.ContextPath{} + for i := 0; i < t.NumField(); i++ { + name := fieldName(v, i, tag) + field := v.Field(i) + if t.Field(i).Anonymous { + ret = append(ret, getAllPaths(field, tag)...) + } else { + ret = append(ret, prefixPaths(getAllPaths(field, tag), name)...) + ret = append(ret, path.New(tag, name)) + } + } + return ret + default: + panic("Encountered types that are not the same when they should be. This is a bug, please file a report") + } +} From c75c66d2bb89f81399906397577f3ce7ac2f243b Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 16 Oct 2019 16:42:42 -0700 Subject: [PATCH 066/877] config: split errors to common Split the common error definitions to config/common. Also differentiate between errors in the source config and in the generated config. --- config/common/errors.go | 24 ++++++++++++++++++++++++ config/v1_0/fcos.go | 9 ++------- config/v1_1_exp/fcos.go | 9 ++------- 3 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 config/common/errors.go diff --git a/config/common/errors.go b/config/common/errors.go new file mode 100644 index 000000000..4bb669aaa --- /dev/null +++ b/config/common/errors.go @@ -0,0 +1,24 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package common + +import ( + "errors" +) + +var ( + ErrInvalidSourceConfig = errors.New("source config is invalid") + ErrInvalidGeneratedConfig = errors.New("config generated was invalid") +) diff --git a/config/v1_0/fcos.go b/config/v1_0/fcos.go index a1631a18c..bbc8d4739 100644 --- a/config/v1_0/fcos.go +++ b/config/v1_0/fcos.go @@ -15,7 +15,6 @@ package v1_0 import ( - "errors" "reflect" base_0_1 "github.com/coreos/fcct/base/v0_1" @@ -30,10 +29,6 @@ import ( "github.com/coreos/vcontext/validate" ) -var ( - ErrInvalidConfig = errors.New("config generated was invalid") -) - type Config struct { common.Common `yaml:",inline"` base_0_1.Config `yaml:",inline"` @@ -74,7 +69,7 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, repo r.Merge(validate.ValidateCustom(cfg, "yaml", unusedKeyCheck)) r.Correlate(contextTree) if r.IsFatal() { - return nil, r, ErrInvalidConfig + return nil, r, common.ErrInvalidSourceConfig } final, translations, err := cfg.Translate() @@ -88,7 +83,7 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, repo r.Merge(second) if r.IsFatal() { - return nil, r, ErrInvalidConfig + return nil, r, common.ErrInvalidGeneratedConfig } outbytes, err := common.Marshal(final, options.Pretty) diff --git a/config/v1_1_exp/fcos.go b/config/v1_1_exp/fcos.go index 4d111c029..b12a0eef5 100644 --- a/config/v1_1_exp/fcos.go +++ b/config/v1_1_exp/fcos.go @@ -15,7 +15,6 @@ package v1_1_exp import ( - "errors" "reflect" base_0_2 "github.com/coreos/fcct/base/v0_2_exp" @@ -30,10 +29,6 @@ import ( "github.com/coreos/vcontext/validate" ) -var ( - ErrInvalidConfig = errors.New("config generated was invalid") -) - type Config struct { common.Common `yaml:",inline"` base_0_2.Config `yaml:",inline"` @@ -74,7 +69,7 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, repo r.Merge(validate.ValidateCustom(cfg, "yaml", unusedKeyCheck)) r.Correlate(contextTree) if r.IsFatal() { - return nil, r, ErrInvalidConfig + return nil, r, common.ErrInvalidSourceConfig } final, translations, err := cfg.Translate() @@ -88,7 +83,7 @@ func TranslateBytes(input []byte, options common.TranslateOptions) ([]byte, repo r.Merge(second) if r.IsFatal() { - return nil, r, ErrInvalidConfig + return nil, r, common.ErrInvalidGeneratedConfig } outbytes, err := common.Marshal(final, options.Pretty) From 841c6847d57e428952c6ffe197ac3919dcc8e761 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Wed, 16 Oct 2019 16:44:15 -0700 Subject: [PATCH 067/877] base/v0_2: add filesystem.with_mount_unit Add a field to automatically generate mount units from filesystem entries. Also update go.mod --- base/v0_2_exp/translate.go | 73 ++++++++++++++++++++++++++++++++++++++ base/v0_2_exp/validate.go | 17 ++++++++- go.mod | 10 +++--- go.sum | 16 ++++++--- 4 files changed, 107 insertions(+), 9 deletions(-) diff --git a/base/v0_2_exp/translate.go b/base/v0_2_exp/translate.go index c85e66ceb..ddf9c248c 100644 --- a/base/v0_2_exp/translate.go +++ b/base/v0_2_exp/translate.go @@ -16,14 +16,34 @@ package v0_2_exp import ( "net/url" + "strings" + "text/template" "github.com/coreos/fcct/translate" + "github.com/coreos/go-systemd/unit" + "github.com/coreos/ignition/v2/config/util" "github.com/coreos/ignition/v2/config/v3_1_experimental/types" "github.com/coreos/vcontext/path" "github.com/vincent-petithory/dataurl" ) +var ( + mountUnitTemplate = template.Must(template.New("unit").Parse(`# Generated by FCCT +[Unit] +Before=local-fs.target +Requires=systemd-fsck@{{.Device}} +After=systemd-fsck@{{.Device}} + +[Mount] +Where={{.Path}} +What={{.Device}} +Type={{.Format}} + +[Install] +RequiredBy=local-fs.target`)) +) + // ToIgn3_0 translates the config to an Ignition config. It also returns the set of translations // it did so paths in the resultant config can be tracked back to their source in the source config. func (c Config) ToIgn3_1() (types.Config, translate.TranslationSet, error) { @@ -35,6 +55,7 @@ func (c Config) ToIgn3_1() (types.Config, translate.TranslationSet, error) { tr.AddCustomTranslator(translateLink) tr.AddCustomTranslator(translateFilesystem) translations := tr.Translate(&c, &ret) + translations.Merge(c.addMountUnits(&ret)) return ret, translations, nil } @@ -114,3 +135,55 @@ func translateFilesystem(from Filesystem) (to types.Filesystem, tm translate.Tra tm.AddIdentity("device", "format", "label", "path", "uuid", "wipeFilesystem") return } + +func (c Config) addMountUnits(ret *types.Config) translate.TranslationSet { + ts := translate.NewTranslationSet("yaml", "json") + if len(c.Storage.Filesystems) == 0 { + return ts + } + unitMap := make(map[string]int, len(ret.Systemd.Units)) + for i, u := range ret.Systemd.Units { + unitMap[u.Name] = i + } + for i, fs := range c.Storage.Filesystems { + if fs.WithMountUnit == nil || !*fs.WithMountUnit { + continue + } + fromPath := path.New("yaml", "storage", "filesystems", i, "with_mount_unit") + newUnit := mountUnitFromFS(fs) + if i, ok := unitMap[unit.UnitNamePathEscape(*fs.Path)+".mount"]; ok { + // user also specified a unit, only set contents and enabled if the existing unit + // is unspecified + u := &ret.Systemd.Units[i] + unitPath := path.New("json", "systemd", "units", i) + if u.Contents == nil { + (*u).Contents = newUnit.Contents + ts.AddTranslation(fromPath, unitPath.Append("contents")) + } + if u.Enabled == nil { + (*u).Enabled = newUnit.Enabled + ts.AddTranslation(fromPath, unitPath.Append("enabled")) + } + } else { + unitPath := path.New("json", "systemd", "units", len(ret.Systemd.Units)) + ret.Systemd.Units = append(ret.Systemd.Units, newUnit) + ts.AddFromCommonSource(fromPath, unitPath, newUnit) + } + } + return ts +} + +func mountUnitFromFS(fs Filesystem) types.Unit { + contents := strings.Builder{} + err := mountUnitTemplate.Execute(&contents, fs) + if err != nil { + panic(err) + } + // unchecked deref of path ok, fs would fail validation otherwise + unitName := unit.UnitNamePathEscape(*fs.Path) + ".mount" + return types.Unit{ + Name: unitName, + Enabled: util.BoolToPtr(true), + Contents: util.StrToPtr(contents.String()), + } +} diff --git a/base/v0_2_exp/validate.go b/base/v0_2_exp/validate.go index d60e41c08..711323ba1 100644 --- a/base/v0_2_exp/validate.go +++ b/base/v0_2_exp/validate.go @@ -22,7 +22,9 @@ import ( ) var ( - ErrInlineAndSource = errors.New("inline cannot be specified if source is specified") + ErrInlineAndSource = errors.New("inline cannot be specified if source is specified") + ErrMountUnitNoPath = errors.New("path is required if with_mount_unit is true") + ErrMountUnitNoFormat = errors.New("format is required if with_mount_unit is true") ) func (f FileContents) Validate(c path.ContextPath) (r report.Report) { @@ -31,3 +33,16 @@ func (f FileContents) Validate(c path.ContextPath) (r report.Report) { } return } + +func (fs Filesystem) Validate(c path.ContextPath) (r report.Report) { + if fs.WithMountUnit == nil || !*fs.WithMountUnit { + return + } + if fs.Path == nil || *fs.Path == "" { + r.AddOnError(c.Append("path"), ErrMountUnitNoPath) + } + if fs.Format == nil || *fs.Format == "" { + r.AddOnError(c.Append("format"), ErrMountUnitNoFormat) + } + return +} diff --git a/go.mod b/go.mod index 06fa513f4..429e510f3 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.12 require ( github.com/coreos/go-semver v0.3.0 - github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/coreos/ignition/v2 v2.0.1 - github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a + github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 github.com/davecgh/go-spew v1.1.1 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb - gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.2.4 // indirect + gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652 ) diff --git a/go.sum b/go.sum index e1ab40f6b..16a55aca3 100644 --- a/go.sum +++ b/go.sum @@ -5,13 +5,13 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/ignition/v2 v2.0.1 h1:aKZARDpl6rDr83Ur/fUoQ3uuSM0VCLH5nTERkLczJFM= github.com/coreos/ignition/v2 v2.0.1/go.mod h1:EJP9Gk/u21BPwWa5nT6yYbm5mO0u8JQAAzFqaxgH7Fg= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= -github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a h1:AdSIR3t2/3oreDLKnsCk8Lu5L3+OKlt9Yu7equBivmE= -github.com/coreos/vcontext v0.0.0-20190605182717-e9c4ffaa1f6a/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= +github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 h1:DjoHHi6+9J7DGYPvBdmszKZLY+ucx2bnA77jf8KIk9M= +github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -34,6 +34,8 @@ github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= @@ -44,7 +46,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae h1:ehhBuCxzgQEGk38YjhFv/97fMIc2JGHZAhAWMmEjmu0= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652 h1:VKvJ/mQ4BgCjZUDggYFxTe0qv9jPMHsZPD4Xt91Y5H4= +gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 67d7c58103478d98f1065de6de9e8c621c147ae7 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 17 Oct 2019 16:13:00 -0700 Subject: [PATCH 068/877] vendor: update go.{mod,sum} Update go.* to pick up new updates. --- go.mod | 5 ++--- go.sum | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 429e510f3..2f85ec99a 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,12 @@ go 1.12 require ( github.com/coreos/go-semver v0.3.0 - github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e github.com/coreos/ignition/v2 v2.0.1 github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 github.com/davecgh/go-spew v1.1.1 // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.3.0 github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.4 // indirect gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652 ) diff --git a/go.sum b/go.sum index 16a55aca3..f4016d64b 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/ignition/v2 v2.0.1 h1:aKZARDpl6rDr83Ur/fUoQ3uuSM0VCLH5nTERkLczJFM= github.com/coreos/ignition/v2 v2.0.1/go.mod h1:EJP9Gk/u21BPwWa5nT6yYbm5mO0u8JQAAzFqaxgH7Fg= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= @@ -34,8 +34,6 @@ github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= @@ -50,8 +48,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae h1:ehhBuCxzgQEGk38YjhFv/97fMIc2JGHZAhAWMmEjmu0= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652 h1:VKvJ/mQ4BgCjZUDggYFxTe0qv9jPMHsZPD4Xt91Y5H4= From fe99a1be3f1fd1011538f51763d41b3d42b9fc70 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Thu, 24 Oct 2019 09:15:25 -0700 Subject: [PATCH 069/877] docs: add docs for storage.fs.with_mount_unit --- docs/configuration-v1_1-exp.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration-v1_1-exp.md b/docs/configuration-v1_1-exp.md index d1406613d..c52c5f166 100644 --- a/docs/configuration-v1_1-exp.md +++ b/docs/configuration-v1_1-exp.md @@ -56,6 +56,7 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s * **_label_** (string): the label of the filesystem. * **_uuid_** (string): the uuid of the filesystem. * **_options_** (list of strings): any additional options to be passed to the format-specific mkfs utility. + * **_with_mount_unit_** (bool): Whether to generate a generic mount unit for this filesystem as well. If a more specific unit is needed, a custom one can be specified in the `systemd.units` section. The unit will be named with the [escaped][systemd-escape] version of the `path`. * **_files_** (list of objects): the list of files to be written. Every file, directory and link must have a unique `path`. * **path** (string): the absolute path to the file. * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `source` must be specified if `overwrite` is true. Defaults to false. @@ -130,3 +131,4 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s [part-types]: http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs [rfc2397]: https://tools.ietf.org/html/rfc2397 +[systemd-escape]: https://www.freedesktop.org/software/systemd/man/systemd-escape.html From 8d2ec7831b645a8c464965c02ad8cb32d8b23fa8 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Thu, 23 Jan 2020 00:08:10 -0500 Subject: [PATCH 070/877] news: add news for 0.3.0 --- NEWS | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/NEWS b/NEWS index 4c5546786..95975f338 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,22 @@ +2020-01-23 FCCT 0.3.0 + + Features: + + - Add v1.1.0-experimental spec + - Add with_mount_unit field to generate mount unit from filesystem entry + + Bug Fixes: + + - Report warnings and errors to stderr, not stdout + - Truncate output file before writing + - Fix line and column reporting + + Misc Changes: + + - Document syntax of inline file contents + - Document usage of published container image + + 2019-07-24 FCCT 0.2.0 Features: From bf7e995b1c2a5fdf20200055e5adbd01c365f0c2 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Thu, 23 Jan 2020 12:08:31 -0500 Subject: [PATCH 071/877] release-checklist: fix caps --- .github/ISSUE_TEMPLATE/release-checklist.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release-checklist.md b/.github/ISSUE_TEMPLATE/release-checklist.md index 1e0953b26..3f96ead1b 100644 --- a/.github/ISSUE_TEMPLATE/release-checklist.md +++ b/.github/ISSUE_TEMPLATE/release-checklist.md @@ -4,7 +4,7 @@ Release checklist: - [ ] Ensure your local copy is up to date with master and your working directory is clean - [ ] Ensure you can sign commits and any yubikeys/smartcards are plugged in - [ ] Run `./tag_release.sh ` - - [ ] Push that tag to Github + - [ ] Push that tag to GitHub - [ ] Run `./build_releases` - [ ] Sign the release artifacts by running ``` @@ -12,5 +12,5 @@ gpg --local-user 0xCDDE268EBB729EC7! --detach-sign --armor ``` for each release artifact. Do not try to sign all of them at once by globbing. If you do, gpg will sign the combination of all the release artifacts instead of each one individually. - - [ ] Create a draft release on Github and upload all the release artifacts and their signatures. Copy and paste the release notes from NEWS here as well. + - [ ] Create a draft release on GitHub and upload all the release artifacts and their signatures. Copy and paste the release notes from NEWS here as well. - [ ] Publish the release From ffa322f0ae247f29657954ccc798303cdbdc56a7 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Thu, 23 Jan 2020 12:09:49 -0500 Subject: [PATCH 072/877] {docs,release-checklist}: add Quay tag pointing to latest release Don't expect users to know the version number of the current release. --- .github/ISSUE_TEMPLATE/release-checklist.md | 3 +++ docs/getting-started.md | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release-checklist.md b/.github/ISSUE_TEMPLATE/release-checklist.md index 3f96ead1b..f9761ccce 100644 --- a/.github/ISSUE_TEMPLATE/release-checklist.md +++ b/.github/ISSUE_TEMPLATE/release-checklist.md @@ -14,3 +14,6 @@ for each release artifact. Do not try to sign all of them at once by globbing. I - [ ] Create a draft release on GitHub and upload all the release artifacts and their signatures. Copy and paste the release notes from NEWS here as well. - [ ] Publish the release + - Update the `release` tag on Quay: + - [ ] Visit the [Quay tags page](https://quay.io/repository/coreos/fcct?tab=tags) and wait for a versioned tag to appear + - [ ] Click the gear next to the tag, select "Add New Tag", enter `release`, and confirm diff --git a/docs/getting-started.md b/docs/getting-started.md index 931b4bf09..444dca497 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -21,17 +21,17 @@ New releases of `fcct` are backwards compatible with old releases unless otherwi #### Container -This example uses podman, but docker can also be used. Substitute v0.2.0 with the desired version. Note that the `latest` tag corresponds with `master` and not with the latest release. +This example uses podman, but docker can also be used. ```bash # Pull the desired version -podman pull quay.io/coreos/fcct:v0.2.0 +podman pull quay.io/coreos/fcct:release # Run fcct using standard in and standard out -podman run -i --rm quay.io/coreos/fcct:v0.2.0 -pretty -strict < your_config.fcc > transpiled_config.ign +podman run -i --rm quay.io/coreos/fcct:release -pretty -strict < your_config.fcc > transpiled_config.ign # Run fcct using files. -podman run --rm -v /path/to/your_config.fcc:/config.fcc:z quay.io/coreos/fcct:v0.2.0 -pretty -strict -input /config.fcc > transpiled_config.ign +podman run --rm -v /path/to/your_config.fcc:/config.fcc:z quay.io/coreos/fcct:release -pretty -strict -input /config.fcc > transpiled_config.ign ``` ### Writing and using Fedora CoreOS Configs From cda00b1d7883b16a3f691538f5147e45cf296ada Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Thu, 23 Jan 2020 23:07:56 -0500 Subject: [PATCH 073/877] vendor: vendor dependencies Help(?) with downstream packaging. --- vendor/github.com/ajeddeloh/go-json/README | 10 + vendor/github.com/ajeddeloh/go-json/decode.go | 1226 +++++++ vendor/github.com/ajeddeloh/go-json/encode.go | 1194 +++++++ vendor/github.com/ajeddeloh/go-json/fold.go | 143 + vendor/github.com/ajeddeloh/go-json/indent.go | 137 + .../github.com/ajeddeloh/go-json/scanner.go | 630 ++++ vendor/github.com/ajeddeloh/go-json/stream.go | 480 +++ vendor/github.com/ajeddeloh/go-json/tags.go | 44 + vendor/github.com/coreos/go-semver/LICENSE | 202 ++ vendor/github.com/coreos/go-semver/NOTICE | 5 + .../coreos/go-semver/semver/semver.go | 296 ++ .../coreos/go-semver/semver/sort.go | 38 + vendor/github.com/coreos/go-systemd/LICENSE | 191 ++ vendor/github.com/coreos/go-systemd/NOTICE | 5 + .../coreos/go-systemd/unit/deserialize.go | 278 ++ .../coreos/go-systemd/unit/escape.go | 116 + .../coreos/go-systemd/unit/option.go | 59 + .../coreos/go-systemd/unit/serialize.go | 75 + vendor/github.com/coreos/ignition/v2/LICENSE | 202 ++ vendor/github.com/coreos/ignition/v2/NOTICE | 5 + .../v2/config/shared/errors/errors.go | 92 + .../v2/config/shared/validations/unit.go | 48 + .../coreos/ignition/v2/config/util/helpers.go | 35 + .../ignition/v2/config/util/interfaces.go | 39 + .../ignition/v2/config/util/parsingErrors.go | 50 + .../ignition/v2/config/util/reflection.go | 55 + .../ignition/v2/config/v3_0/types/ca.go | 29 + .../ignition/v2/config/v3_0/types/config.go | 26 + .../ignition/v2/config/v3_0/types/device.go | 25 + .../v2/config/v3_0/types/directory.go | 31 + .../ignition/v2/config/v3_0/types/disk.go | 134 + .../ignition/v2/config/v3_0/types/file.go | 70 + .../v2/config/v3_0/types/filesystem.go | 103 + .../ignition/v2/config/v3_0/types/ignition.go | 54 + .../ignition/v2/config/v3_0/types/mode.go | 26 + .../ignition/v2/config/v3_0/types/node.go | 58 + .../v2/config/v3_0/types/partition.go | 88 + .../ignition/v2/config/v3_0/types/passwd.go | 23 + .../ignition/v2/config/v3_0/types/path.go | 42 + .../ignition/v2/config/v3_0/types/raid.go | 55 + .../ignition/v2/config/v3_0/types/schema.go | 201 ++ .../ignition/v2/config/v3_0/types/storage.go | 70 + .../ignition/v2/config/v3_0/types/unit.go | 83 + .../ignition/v2/config/v3_0/types/url.go | 57 + .../v2/config/v3_0/types/verification.go | 69 + .../v2/config/v3_1_experimental/types/ca.go | 29 + .../config/v3_1_experimental/types/config.go | 27 + .../config/v3_1_experimental/types/device.go | 25 + .../v3_1_experimental/types/directory.go | 31 + .../v2/config/v3_1_experimental/types/disk.go | 134 + .../v2/config/v3_1_experimental/types/file.go | 70 + .../v3_1_experimental/types/filesystem.go | 103 + .../v3_1_experimental/types/ignition.go | 54 + .../v2/config/v3_1_experimental/types/mode.go | 26 + .../v2/config/v3_1_experimental/types/node.go | 58 + .../v3_1_experimental/types/partition.go | 88 + .../config/v3_1_experimental/types/passwd.go | 23 + .../v2/config/v3_1_experimental/types/path.go | 42 + .../v2/config/v3_1_experimental/types/raid.go | 55 + .../config/v3_1_experimental/types/schema.go | 210 ++ .../config/v3_1_experimental/types/storage.go | 70 + .../v2/config/v3_1_experimental/types/unit.go | 83 + .../v2/config/v3_1_experimental/types/url.go | 57 + .../v3_1_experimental/types/verification.go | 69 + .../ignition/v2/config/validate/validate.go | 118 + vendor/github.com/coreos/vcontext/LICENSE | 202 ++ .../github.com/coreos/vcontext/json/json.go | 64 + .../github.com/coreos/vcontext/path/path.go | 85 + .../coreos/vcontext/report/report.go | 153 + .../github.com/coreos/vcontext/tree/tree.go | 216 ++ .../coreos/vcontext/validate/validate.go | 149 + .../github.com/coreos/vcontext/yaml/yaml.go | 82 + vendor/github.com/davecgh/go-spew/LICENSE | 15 + .../github.com/davecgh/go-spew/spew/bypass.go | 145 + .../davecgh/go-spew/spew/bypasssafe.go | 38 + .../github.com/davecgh/go-spew/spew/common.go | 341 ++ .../github.com/davecgh/go-spew/spew/config.go | 306 ++ vendor/github.com/davecgh/go-spew/spew/doc.go | 211 ++ .../github.com/davecgh/go-spew/spew/dump.go | 509 +++ .../github.com/davecgh/go-spew/spew/format.go | 419 +++ .../github.com/davecgh/go-spew/spew/spew.go | 148 + vendor/github.com/pmezard/go-difflib/LICENSE | 27 + .../pmezard/go-difflib/difflib/difflib.go | 772 +++++ vendor/github.com/stretchr/testify/LICENSE | 21 + .../testify/assert/assertion_format.go | 484 +++ .../testify/assert/assertion_format.go.tmpl | 5 + .../testify/assert/assertion_forward.go | 956 ++++++ .../testify/assert/assertion_forward.go.tmpl | 5 + .../stretchr/testify/assert/assertions.go | 1416 ++++++++ .../github.com/stretchr/testify/assert/doc.go | 45 + .../stretchr/testify/assert/errors.go | 10 + .../testify/assert/forward_assertions.go | 16 + .../testify/assert/http_assertions.go | 143 + .../vincent-petithory/dataurl/LICENSE | 20 + .../vincent-petithory/dataurl/README.md | 81 + .../vincent-petithory/dataurl/dataurl.go | 280 ++ .../vincent-petithory/dataurl/doc.go | 28 + .../vincent-petithory/dataurl/lex.go | 521 +++ .../vincent-petithory/dataurl/rfc2396.go | 130 + .../vincent-petithory/dataurl/wercker.yml | 1 + vendor/gopkg.in/yaml.v3/.travis.yml | 15 + vendor/gopkg.in/yaml.v3/LICENSE | 50 + vendor/gopkg.in/yaml.v3/NOTICE | 13 + vendor/gopkg.in/yaml.v3/README.md | 150 + vendor/gopkg.in/yaml.v3/apic.go | 746 ++++ vendor/gopkg.in/yaml.v3/decode.go | 931 +++++ vendor/gopkg.in/yaml.v3/emitterc.go | 1992 +++++++++++ vendor/gopkg.in/yaml.v3/encode.go | 542 +++ vendor/gopkg.in/yaml.v3/go.mod | 5 + vendor/gopkg.in/yaml.v3/parserc.go | 1210 +++++++ vendor/gopkg.in/yaml.v3/readerc.go | 434 +++ vendor/gopkg.in/yaml.v3/resolve.go | 326 ++ vendor/gopkg.in/yaml.v3/scannerc.go | 2990 +++++++++++++++++ vendor/gopkg.in/yaml.v3/sorter.go | 134 + vendor/gopkg.in/yaml.v3/writerc.go | 48 + vendor/gopkg.in/yaml.v3/yaml.go | 662 ++++ vendor/gopkg.in/yaml.v3/yamlh.go | 803 +++++ vendor/gopkg.in/yaml.v3/yamlprivateh.go | 198 ++ vendor/modules.txt | 30 + 119 files changed, 27594 insertions(+) create mode 100644 vendor/github.com/ajeddeloh/go-json/README create mode 100644 vendor/github.com/ajeddeloh/go-json/decode.go create mode 100644 vendor/github.com/ajeddeloh/go-json/encode.go create mode 100644 vendor/github.com/ajeddeloh/go-json/fold.go create mode 100644 vendor/github.com/ajeddeloh/go-json/indent.go create mode 100644 vendor/github.com/ajeddeloh/go-json/scanner.go create mode 100644 vendor/github.com/ajeddeloh/go-json/stream.go create mode 100644 vendor/github.com/ajeddeloh/go-json/tags.go create mode 100644 vendor/github.com/coreos/go-semver/LICENSE create mode 100644 vendor/github.com/coreos/go-semver/NOTICE create mode 100644 vendor/github.com/coreos/go-semver/semver/semver.go create mode 100644 vendor/github.com/coreos/go-semver/semver/sort.go create mode 100644 vendor/github.com/coreos/go-systemd/LICENSE create mode 100644 vendor/github.com/coreos/go-systemd/NOTICE create mode 100644 vendor/github.com/coreos/go-systemd/unit/deserialize.go create mode 100644 vendor/github.com/coreos/go-systemd/unit/escape.go create mode 100644 vendor/github.com/coreos/go-systemd/unit/option.go create mode 100644 vendor/github.com/coreos/go-systemd/unit/serialize.go create mode 100644 vendor/github.com/coreos/ignition/v2/LICENSE create mode 100644 vendor/github.com/coreos/ignition/v2/NOTICE create mode 100644 vendor/github.com/coreos/ignition/v2/config/shared/errors/errors.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/shared/validations/unit.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/util/helpers.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/util/interfaces.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/util/parsingErrors.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/util/reflection.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/ca.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/config.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/device.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/directory.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/disk.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/file.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/filesystem.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/ignition.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/mode.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/node.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/partition.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/passwd.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/path.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/raid.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/schema.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/storage.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/unit.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/url.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_0/types/verification.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/ca.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/config.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/device.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/directory.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/disk.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/file.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/filesystem.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/ignition.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/mode.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/node.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/partition.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/passwd.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/path.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/raid.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/schema.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/storage.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/unit.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/url.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/v3_1_experimental/types/verification.go create mode 100644 vendor/github.com/coreos/ignition/v2/config/validate/validate.go create mode 100644 vendor/github.com/coreos/vcontext/LICENSE create mode 100644 vendor/github.com/coreos/vcontext/json/json.go create mode 100644 vendor/github.com/coreos/vcontext/path/path.go create mode 100644 vendor/github.com/coreos/vcontext/report/report.go create mode 100644 vendor/github.com/coreos/vcontext/tree/tree.go create mode 100644 vendor/github.com/coreos/vcontext/validate/validate.go create mode 100644 vendor/github.com/coreos/vcontext/yaml/yaml.go create mode 100644 vendor/github.com/davecgh/go-spew/LICENSE create mode 100644 vendor/github.com/davecgh/go-spew/spew/bypass.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/bypasssafe.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/common.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/config.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/doc.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/dump.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/format.go create mode 100644 vendor/github.com/davecgh/go-spew/spew/spew.go create mode 100644 vendor/github.com/pmezard/go-difflib/LICENSE create mode 100644 vendor/github.com/pmezard/go-difflib/difflib/difflib.go create mode 100644 vendor/github.com/stretchr/testify/LICENSE create mode 100644 vendor/github.com/stretchr/testify/assert/assertion_format.go create mode 100644 vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl create mode 100644 vendor/github.com/stretchr/testify/assert/assertion_forward.go create mode 100644 vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl create mode 100644 vendor/github.com/stretchr/testify/assert/assertions.go create mode 100644 vendor/github.com/stretchr/testify/assert/doc.go create mode 100644 vendor/github.com/stretchr/testify/assert/errors.go create mode 100644 vendor/github.com/stretchr/testify/assert/forward_assertions.go create mode 100644 vendor/github.com/stretchr/testify/assert/http_assertions.go create mode 100644 vendor/github.com/vincent-petithory/dataurl/LICENSE create mode 100644 vendor/github.com/vincent-petithory/dataurl/README.md create mode 100644 vendor/github.com/vincent-petithory/dataurl/dataurl.go create mode 100644 vendor/github.com/vincent-petithory/dataurl/doc.go create mode 100644 vendor/github.com/vincent-petithory/dataurl/lex.go create mode 100644 vendor/github.com/vincent-petithory/dataurl/rfc2396.go create mode 100644 vendor/github.com/vincent-petithory/dataurl/wercker.yml create mode 100644 vendor/gopkg.in/yaml.v3/.travis.yml create mode 100644 vendor/gopkg.in/yaml.v3/LICENSE create mode 100644 vendor/gopkg.in/yaml.v3/NOTICE create mode 100644 vendor/gopkg.in/yaml.v3/README.md create mode 100644 vendor/gopkg.in/yaml.v3/apic.go create mode 100644 vendor/gopkg.in/yaml.v3/decode.go create mode 100644 vendor/gopkg.in/yaml.v3/emitterc.go create mode 100644 vendor/gopkg.in/yaml.v3/encode.go create mode 100644 vendor/gopkg.in/yaml.v3/go.mod create mode 100644 vendor/gopkg.in/yaml.v3/parserc.go create mode 100644 vendor/gopkg.in/yaml.v3/readerc.go create mode 100644 vendor/gopkg.in/yaml.v3/resolve.go create mode 100644 vendor/gopkg.in/yaml.v3/scannerc.go create mode 100644 vendor/gopkg.in/yaml.v3/sorter.go create mode 100644 vendor/gopkg.in/yaml.v3/writerc.go create mode 100644 vendor/gopkg.in/yaml.v3/yaml.go create mode 100644 vendor/gopkg.in/yaml.v3/yamlh.go create mode 100644 vendor/gopkg.in/yaml.v3/yamlprivateh.go create mode 100644 vendor/modules.txt diff --git a/vendor/github.com/ajeddeloh/go-json/README b/vendor/github.com/ajeddeloh/go-json/README new file mode 100644 index 000000000..0f53d3450 --- /dev/null +++ b/vendor/github.com/ajeddeloh/go-json/README @@ -0,0 +1,10 @@ +This is a fork of go's encoding/json library. It adds the a third target for unmarshalling, json.Node. +Unmarshalling to a Node behaves similarilarly to unmarshalling to an interface{}, except it also records +the offsets for the start and end of the value that was unmarshalled and, if the value was part of a json +object, it also records the offsets of the start and end of the object's key. The Value field of the Node +will be unmarshalled to the same types as if it were an interface{}, except in the case of arrays and +objects. In those case it will be unmarshalled to a []Node or map[string]Node instead []interface{} or +map[string]interface{} for arrays and objects, respectively. + +There are two branchs, go15 and go16. go15 contains the modified go1.5 library and go16 contains the +modified go1.6 library. diff --git a/vendor/github.com/ajeddeloh/go-json/decode.go b/vendor/github.com/ajeddeloh/go-json/decode.go new file mode 100644 index 000000000..1dc2fdf0d --- /dev/null +++ b/vendor/github.com/ajeddeloh/go-json/decode.go @@ -0,0 +1,1226 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "errors" + "fmt" + "reflect" + "runtime" + "strconv" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. +// +// Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by Marshal (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// To unmarshal a JSON array into a slice, Unmarshal resets the slice to nil +// and then appends each element to the slice. +// +// To unmarshal a JSON object into a map, Unmarshal replaces the map +// with an empty map and then adds key-value pairs from the object to +// the map. +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshalling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +// +func Unmarshal(data []byte, v interface{}) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by objects +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes +} + +func (e *UnmarshalTypeError) Error() string { + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// (No longer used; kept for compatibility.) +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +type Node struct { + Start int + End int + KeyStart int // Only value if a member of a struct + KeyEnd int + Value interface{} +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Ptr { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v interface{}) (err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + d.value(rv) + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + savedError error + useNumber bool +} + +// errPhase is used for errors that should not happen unless +// there is a bug in the JSON decoder or something is editing +// the data slice while the decoder executes. +var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + return d +} + +// error aborts the decoding by panicking with err. +func (d *decodeState) error(err error) { + panic(err) +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + d.savedError = err + } +} + +// next cuts off and returns the next full JSON value in d.data[d.off:]. +// The next value is known to be an object or array, not a literal. +func (d *decodeState) next() []byte { + c := d.data[d.off] + item, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // Our scanner has seen the opening brace/bracket + // and thinks we're still in the middle of the object. + // invent a closing brace/bracket to get it out. + if c == '{' { + d.scan.step(&d.scan, '}') + } else { + d.scan.step(&d.scan, ']') + } + + return item +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +// It updates d.off and returns the new scan code. +func (d *decodeState) scanWhile(op int) int { + var newOp int + for { + if d.off >= len(d.data) { + newOp = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } else { + c := int(d.data[d.off]) + d.off++ + newOp = d.scan.step(&d.scan, c) + } + if newOp != op { + break + } + } + return newOp +} + +// value decodes a JSON value from d.data[d.off:] into the value. +// it updates d.off to point past the decoded value. +func (d *decodeState) value(v reflect.Value) { + if !v.IsValid() { + _, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // d.scan thinks we're still at the beginning of the item. + // Feed in an empty string - the shortest, simplest value - + // so that it knows we got to the end of the value. + if d.scan.redo { + // rewind. + d.scan.redo = false + d.scan.step = stateBeginValue + } + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + + n := len(d.scan.parseState) + if n > 0 && d.scan.parseState[n-1] == parseObjectKey { + // d.scan thinks we just read an object key; finish the object + d.scan.step(&d.scan, ':') + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '}') + } + + return + } + + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(v) + + case scanBeginObject: + d.object(v) + + case scanBeginLiteral: + d.literal(v) + } +} + +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() interface{} { + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(reflect.Value{}) + + case scanBeginObject: + d.object(reflect.Value{}) + + case scanBeginLiteral: + switch v := d.literalInterface().(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + v = v.Elem() + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into the value v. +// the first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) { + // Check for unmarshaler. + u, ut, pv := d.indirect(v, false) + if u != nil { + d.off-- + err := u.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) + d.off-- + d.next() + return + } + + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + v.Set(reflect.ValueOf(d.arrayInterface())) + return + } + // Otherwise it's invalid. + fallthrough + default: + if v.Type() == reflect.TypeOf(Node{}) { + // Decoding to Node? Switch to that code + v.Set(reflect.ValueOf(d.arrayNode())) + return + } + d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) + d.off-- + d.next() + return + case reflect.Array: + case reflect.Slice: + break + } + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + // Get element of array, growing if necessary. + if v.Kind() == reflect.Slice { + // Grow slice if necessary + if i >= v.Cap() { + newcap := v.Cap() + v.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) + reflect.Copy(newv, v) + v.Set(newv) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + d.value(v.Index(i)) + } else { + // Ran out of fixed array: skip. + d.value(reflect.Value{}) + } + i++ + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + + if i < v.Len() { + if v.Kind() == reflect.Array { + // Array. Zero the rest. + z := reflect.Zero(v.Type().Elem()) + for ; i < v.Len(); i++ { + v.Index(i).Set(z) + } + } else { + v.SetLen(i) + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } +} + +var nullLiteral = []byte("null") + +// object consumes an object from d.data[d.off-1:], decoding into the value v. +// the first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) { + // Check for unmarshaler. + u, ut, pv := d.indirect(v, false) + if u != nil { + d.off-- + err := u.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.off-- + d.next() // skip over { } in input + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + v.Set(reflect.ValueOf(d.objectInterface())) + return + } else if v.Type() == reflect.TypeOf(Node{}) { + // Decoding to Node? Switch to that code + v.Set(reflect.ValueOf(d.objectNode())) + return + } + + // Check type of target: struct or map[string]T + switch v.Kind() { + case reflect.Map: + // map must have string kind + t := v.Type() + if t.Key().Kind() != reflect.String { + d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.off-- + d.next() // skip over { } in input + return + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + + default: + d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.off-- + d.next() // skip over { } in input + return + } + + var mapElem reflect.Value + + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquoteBytes(item) + if !ok { + d.error(errPhase) + } + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := v.Type().Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.Set(reflect.Zero(elemType)) + } + subv = mapElem + } else { + var f *field + fields := cachedTypeFields(v.Type()) + for i := range fields { + ff := &fields[i] + if bytes.Equal(ff.nameBytes, key) { + f = ff + break + } + if f == nil && ff.equalFold(ff.nameBytes, key) { + f = ff + } + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + } + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + if destring { + switch qv := d.valueQuoted().(type) { + case nil: + d.literalStore(nullLiteral, subv, false) + case string: + d.literalStore([]byte(qv), subv, true) + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) + } + } else { + d.value(subv) + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kv := reflect.ValueOf(key).Convert(v.Type().Key()) + v.SetMapIndex(kv, subv) + } + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } +} + +// literal consumes a literal from d.data[d.off-1:], decoding into the value v. +// The first byte of the literal has been read already +// (that's how the caller knows it's a literal). +func (d *decodeState) literal(v reflect.Value) { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + + d.literalStore(d.data[start:d.off], v, false) +} + +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (interface{}, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} + } + return f, nil +} + +var numberType = reflect.TypeOf(Number("")) + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { + // Check for unmarshaler. + if len(item) == 0 { + //Empty string given + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return + } + wantptr := item[0] == 'n' // null + u, ut, pv := d.indirect(v, wantptr) + if u != nil { + err := u.UnmarshalJSON(item) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + } + return + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + err := ut.UnmarshalText(s) + if err != nil { + d.error(err) + } + return + } + + v = pv + + switch c := item[0]; c { + case 'n': // null + switch v.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + v.Set(reflect.Zero(v.Type())) + // otherwise, ignore null for primitives/string + } + case 't', 'f': // true, false + value := c == 't' + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) + } + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + case reflect.Slice: + if v.Type().Elem().Kind() != reflect.Uint8 { + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.ValueOf(b[0:n])) + case reflect.String: + v.SetString(string(s)) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + } + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + s := string(item) + switch v.Kind() { + default: + if v.Kind() == reflect.String && v.Type() == numberType { + v.SetString(s) + break + } + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) + } + case reflect.Interface: + n, err := d.convertNumber(s) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + break + } + v.SetFloat(n) + } + } +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() interface{} { + switch d.scanWhile(scanSkipSpace) { + default: + d.error(errPhase) + panic("unreachable") + case scanBeginArray: + return d.arrayInterface() + case scanBeginObject: + return d.objectInterface() + case scanBeginLiteral: + return d.literalInterface() + } +} + +// valueNode is like valueInterface but returns a wrapped version that +// contains metadata about where it decoded from +func (d *decodeState) valueNode() Node { + switch d.scanWhile(scanSkipSpace) { + default: + d.error(errPhase) + panic("unreachable") + case scanBeginArray: + return d.arrayNode() + case scanBeginObject: + return d.objectNode() + case scanBeginLiteral: + return d.literalNode() + } +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []interface{} { + var v = make([]interface{}, 0) + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v = append(v, d.valueInterface()) + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + return v +} + +// arrayNode is like arrayInterface but returns Node. +func (d *decodeState) arrayNode() Node { + var v = make([]Node, 0) + node := Node{ + Start: d.off, + Value: v, + } + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v = append(v, d.valueNode()) + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + node.Value = v + node.End = d.off - 1 + return node +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + return m +} + +// objectNode is like object but returns Node. +func (d *decodeState) objectNode() Node { + m := make(map[string]Node) + node := Node{ + Start: d.off, + } + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + keyEnd := d.off - 1 + key, ok := unquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + val := d.valueNode() + val.KeyStart = start + val.KeyEnd = keyEnd + m[key] = val + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + node.Value = m + node.End = d.off - 1 + return node +} + +// literalInterface is like literal but returns an interface value. +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + d.error(errPhase) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + d.error(errPhase) + } + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + } +} + +func (d *decodeState) literalNode() Node { + start := d.off - 1 + // Can just use the interface version since this has no children + node := Node{ + Start: start, + Value: d.literalInterface(), + } + node.End = d.off - 1 + return node +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + r, err := strconv.ParseUint(string(s[2:6]), 16, 64) + if err != nil { + return -1 + } + return rune(r) +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/vendor/github.com/ajeddeloh/go-json/encode.go b/vendor/github.com/ajeddeloh/go-json/encode.go new file mode 100644 index 000000000..90782deb7 --- /dev/null +++ b/vendor/github.com/ajeddeloh/go-json/encode.go @@ -0,0 +1,1194 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON objects as defined in +// RFC 4627. The mapping between JSON objects and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// https://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "math" + "reflect" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and Number values encode as JSON numbers. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" +// to keep some browsers from misinterpreting JSON output as HTML. +// Ampersand "&" is also escaped to "\u0026" for the same reason. +// +// Array and slice values encode as JSON arrays, except that +// []byte encodes as a base64-encoded string, and a nil slice +// encodes as the null JSON object. +// +// Struct values encode as JSON objects. Each exported struct field +// becomes a member of the object unless +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option. +// The empty values are false, 0, any +// nil pointer or interface value, and any array, slice, map, or string of +// length zero. The object's default key string is the struct field name +// but can be specified in the struct field's tag value. The "json" key in +// the struct field's tag value is the key name, followed by an optional comma +// and options. Examples: +// +// // Field is ignored by this package. +// Field int `json:"-"` +// +// // Field appears in JSON as key "myName". +// Field int `json:"myName"` +// +// // Field appears in JSON as key "myName" and +// // the field is omitted from the object if its value is empty, +// // as defined above. +// Field int `json:"myName,omitempty"` +// +// // Field appears in JSON as key "Field" (the default), but +// // the field is skipped if empty. +// // Note the leading comma. +// Field int `json:",omitempty"` +// +// The "string" option signals that a field is stored as JSON inside a +// JSON-encoded string. It applies only to fields of string, floating point, +// integer, or boolean types. This extra level of encoding is sometimes used +// when communicating with JavaScript programs: +// +// Int64String int64 `json:",string"` +// +// The key name will be used if it's a non-empty string consisting of +// only Unicode letters, digits, dollar signs, percent signs, hyphens, +// underscores and slashes. +// +// Anonymous struct fields are usually marshaled as if their inner exported fields +// were fields in the outer struct, subject to the usual Go visibility rules amended +// as described in the next paragraph. +// An anonymous struct field with a name given in its JSON tag is treated as +// having that name, rather than being anonymous. +// An anonymous struct field of interface type is treated the same as having +// that type as its name, rather than being anonymous. +// +// The Go visibility rules for struct fields are amended for JSON when +// deciding which field to marshal or unmarshal. If there are +// multiple fields at the same level, and that level is the least +// nested (and would therefore be the nesting level selected by the +// usual Go rules), the following extra rules apply: +// +// 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, +// even if there are multiple untagged fields that would otherwise conflict. +// 2) If there is exactly one field (tagged or not according to the first rule), that is selected. +// 3) Otherwise there are multiple fields, and all are ignored; no error occurs. +// +// Handling of anonymous struct fields is new in Go 1.1. +// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of +// an anonymous struct field in both current and earlier versions, give the field +// a JSON tag of "-". +// +// Map values encode as JSON objects. +// The map's key type must be string; the map keys are used as JSON object +// keys, subject to the UTF-8 coercion described for string values above. +// +// Pointer values encode as the value pointed to. +// A nil pointer encodes as the null JSON object. +// +// Interface values encode as the value contained in the interface. +// A nil interface value encodes as the null JSON object. +// +// Channel, complex, and function values cannot be encoded in JSON. +// Attempting to encode such a value causes Marshal to return +// an UnsupportedTypeError. +// +// JSON cannot represent cyclic data structures and Marshal does not +// handle them. Passing cyclic structures to Marshal will result in +// an infinite recursion. +// +func Marshal(v interface{}) ([]byte, error) { + e := &encodeState{} + err := e.marshal(v) + if err != nil { + return nil, err + } + return e.Bytes(), nil +} + +// MarshalIndent is like Marshal but applies Indent to format the output. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + b, err := Marshal(v) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = Indent(&buf, b, prefix, indent) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 +// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 +// so that the JSON will be safe to embed inside HTML