From d21f39eeebd3586e7faf4d83c7a8e12b6e04c82e Mon Sep 17 00:00:00 2001 From: Victor Häggqvist Date: Tue, 21 Apr 2020 09:34:44 +0200 Subject: replace ini --- vendor/github.com/gocarina/gocsv/.travis.yml | 1 + vendor/github.com/gocarina/gocsv/README.md | 8 +- vendor/github.com/gocarina/gocsv/csv.go | 166 +++++++++++++++++++- vendor/github.com/gocarina/gocsv/decode.go | 191 +++++++++++++++++++---- vendor/github.com/gocarina/gocsv/encode.go | 8 + vendor/github.com/gocarina/gocsv/go.mod | 3 + vendor/github.com/gocarina/gocsv/reflect.go | 57 +++++-- vendor/github.com/gocarina/gocsv/types.go | 80 +++++----- vendor/github.com/gocarina/gocsv/unmarshaller.go | 35 ++++- 9 files changed, 451 insertions(+), 98 deletions(-) create mode 100644 vendor/github.com/gocarina/gocsv/.travis.yml create mode 100644 vendor/github.com/gocarina/gocsv/go.mod (limited to 'vendor/github.com/gocarina/gocsv') diff --git a/vendor/github.com/gocarina/gocsv/.travis.yml b/vendor/github.com/gocarina/gocsv/.travis.yml new file mode 100644 index 0000000..4f2ee4d --- /dev/null +++ b/vendor/github.com/gocarina/gocsv/.travis.yml @@ -0,0 +1 @@ +language: go diff --git a/vendor/github.com/gocarina/gocsv/README.md b/vendor/github.com/gocarina/gocsv/README.md index c3fae62..e606ac7 100644 --- a/vendor/github.com/gocarina/gocsv/README.md +++ b/vendor/github.com/gocarina/gocsv/README.md @@ -36,8 +36,9 @@ package main import ( "fmt" - "gocsv" "os" + + "github.com/gocarina/gocsv" ) type Client struct { // Our example struct, you can use "-" to ignore a field @@ -104,10 +105,7 @@ func (date *DateTime) String() (string) { // Convert the CSV string as internal date func (date *DateTime) UnmarshalCSV(csv string) (err error) { date.Time, err = time.Parse("20060201", csv) - if err != nil { - return err - } - return nil + return err } type Client struct { // Our example struct with a custom type (DateTime) diff --git a/vendor/github.com/gocarina/gocsv/csv.go b/vendor/github.com/gocarina/gocsv/csv.go index 2e336ea..22846c0 100644 --- a/vendor/github.com/gocarina/gocsv/csv.go +++ b/vendor/github.com/gocarina/gocsv/csv.go @@ -14,6 +14,7 @@ import ( "os" "reflect" "strings" + "sync" ) // FailIfUnmatchedStructTags indicates whether it is considered an error when there is an unmatched @@ -28,9 +29,33 @@ var FailIfDoubleHeaderNames = false // headers per their alignment in the struct definition. var ShouldAlignDuplicateHeadersWithStructFieldOrder = false +// TagName defines key in the struct field's tag to scan +var TagName = "csv" + // TagSeparator defines seperator string for multiple csv tags in struct fields var TagSeparator = "," +// Normalizer is a function that takes and returns a string. It is applied to +// struct and header field values before they are compared. It can be used to alter +// names for comparison. For instance, you could allow case insensitive matching +// or convert '-' to '_'. +type Normalizer func(string) string + +type ErrorHandler func(*csv.ParseError) bool + +// normalizeName function initially set to a nop Normalizer. +var normalizeName = DefaultNameNormalizer() + +// DefaultNameNormalizer is a nop Normalizer. +func DefaultNameNormalizer() Normalizer { return func(s string) string { return s } } + +// SetHeaderNormalizer sets the normalizer used to normalize struct and header field names. +func SetHeaderNormalizer(f Normalizer) { + normalizeName = f + // Need to clear the cache hen the header normalizer changes. + structInfoCache = sync.Map{} +} + // -------------------------------------------------------------------------- // CSVWriter used to format CSV @@ -117,7 +142,7 @@ func Marshal(in interface{}, out io.Writer) (err error) { return writeTo(writer, in, false) } -// Marshal returns the CSV in writer from the interface. +// MarshalWithoutHeaders returns the CSV in writer from the interface. func MarshalWithoutHeaders(in interface{}, out io.Writer) (err error) { writer := getCSVWriter(out) return writeTo(writer, in, true) @@ -146,6 +171,11 @@ func UnmarshalFile(in *os.File, out interface{}) error { return Unmarshal(in, out) } +// UnmarshalFile parses the CSV from the file in the interface. +func UnmarshalFileWithErrorHandler(in *os.File, errHandler ErrorHandler, out interface{}) error { + return UnmarshalWithErrorHandler(in, errHandler, out) +} + // UnmarshalString parses the CSV from the string in the interface. func UnmarshalString(in string, out interface{}) error { return Unmarshal(strings.NewReader(in), out) @@ -158,7 +188,22 @@ func UnmarshalBytes(in []byte, out interface{}) error { // Unmarshal parses the CSV from the reader in the interface. func Unmarshal(in io.Reader, out interface{}) error { - return readTo(newDecoder(in), out) + return readTo(newSimpleDecoderFromReader(in), out) +} + +// Unmarshal parses the CSV from the reader in the interface. +func UnmarshalWithErrorHandler(in io.Reader, errHandle ErrorHandler, out interface{}) error { + return readToWithErrorHandler(newSimpleDecoderFromReader(in), errHandle, out) +} + +// UnmarshalWithoutHeaders parses the CSV from the reader in the interface. +func UnmarshalWithoutHeaders(in io.Reader, out interface{}) error { + return readToWithoutHeaders(newSimpleDecoderFromReader(in), out) +} + +// UnmarshalCSVWithoutHeaders parses a headerless CSV with passed in CSV reader +func UnmarshalCSVWithoutHeaders(in CSVReader, out interface{}) error { + return readToWithoutHeaders(csvDecoder{in}, out) } // UnmarshalDecoder parses the CSV from the decoder in the interface @@ -177,7 +222,16 @@ func UnmarshalToChan(in io.Reader, c interface{}) error { if c == nil { return fmt.Errorf("goscv: channel is %v", c) } - return readEach(newDecoder(in), c) + return readEach(newSimpleDecoderFromReader(in), c) +} + +// UnmarshalToChanWithoutHeaders parses the CSV from the reader and send each value in the chan c. +// The channel must have a concrete type. +func UnmarshalToChanWithoutHeaders(in io.Reader, c interface{}) error { + if c == nil { + return fmt.Errorf("goscv: channel is %v", c) + } + return readEachWithoutHeaders(newSimpleDecoderFromReader(in), c) } // UnmarshalDecoderToChan parses the CSV from the decoder and send each value in the chan c. @@ -269,9 +323,88 @@ func UnmarshalStringToCallback(in string, c interface{}) (err error) { return UnmarshalToCallback(strings.NewReader(in), c) } +// UnmarshalToCallbackWithError parses the CSV from the reader and +// send each value to the given func f. +// +// If func returns error, it will stop processing, drain the +// parser and propagate the error to caller. +// +// The func must look like func(Struct) error. +func UnmarshalToCallbackWithError(in io.Reader, f interface{}) error { + valueFunc := reflect.ValueOf(f) + t := reflect.TypeOf(f) + if t.NumIn() != 1 { + return fmt.Errorf("the given function must have exactly one parameter") + } + if t.NumOut() != 1 { + return fmt.Errorf("the given function must have exactly one return value") + } + if !isErrorType(t.Out(0)) { + return fmt.Errorf("the given function must only return error.") + } + + cerr := make(chan error) + c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0) + go func() { + cerr <- UnmarshalToChan(in, c.Interface()) + }() + + var fErr error + for { + select { + case err := <-cerr: + if err != nil { + return err + } + return fErr + default: + } + v, notClosed := c.Recv() + if !notClosed || v.Interface() == nil { + break + } + + // callback f has already returned an error, stop processing but keep draining the chan c + if fErr != nil { + continue + } + + results := valueFunc.Call([]reflect.Value{v}) + + // If the callback f returns an error, stores it and returns it in future. + errValue := results[0].Interface() + if errValue != nil { + fErr = errValue.(error) + } + } + return fErr +} + +// UnmarshalBytesToCallbackWithError parses the CSV from the bytes and +// send each value to the given func f. +// +// If func returns error, it will stop processing, drain the +// parser and propagate the error to caller. +// +// The func must look like func(Struct) error. +func UnmarshalBytesToCallbackWithError(in []byte, f interface{}) error { + return UnmarshalToCallbackWithError(bytes.NewReader(in), f) +} + +// UnmarshalStringToCallbackWithError parses the CSV from the string and +// send each value to the given func f. +// +// If func returns error, it will stop processing, drain the +// parser and propagate the error to caller. +// +// The func must look like func(Struct) error. +func UnmarshalStringToCallbackWithError(in string, c interface{}) (err error) { + return UnmarshalToCallbackWithError(strings.NewReader(in), c) +} + // CSVToMap creates a simple map from a CSV of 2 columns. func CSVToMap(in io.Reader) (map[string]string, error) { - decoder := newDecoder(in) + decoder := newSimpleDecoderFromReader(in) header, err := decoder.getCSVRow() if err != nil { return nil, err @@ -317,3 +450,28 @@ func CSVToMaps(reader io.Reader) ([]map[string]string, error) { } return rows, nil } + +// CSVToChanMaps parses the CSV from the reader and send a dictionary in the chan c, using the header row as the keys. +func CSVToChanMaps(reader io.Reader, c chan<- map[string]string) error { + r := csv.NewReader(reader) + var header []string + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return err + } + if header == nil { + header = record + } else { + dict := map[string]string{} + for i := range header { + dict[header[i]] = record[i] + } + c <- dict + } + } + return nil +} diff --git a/vendor/github.com/gocarina/gocsv/decode.go b/vendor/github.com/gocarina/gocsv/decode.go index 980d7f7..a832e84 100644 --- a/vendor/github.com/gocarina/gocsv/decode.go +++ b/vendor/github.com/gocarina/gocsv/decode.go @@ -16,26 +16,7 @@ type Decoder interface { // SimpleDecoder . type SimpleDecoder interface { getCSVRow() ([]string, error) -} - -type decoder struct { - in io.Reader - csvDecoder *csvDecoder -} - -func newDecoder(in io.Reader) *decoder { - return &decoder{in: in} -} - -func (decode *decoder) getCSVRows() ([][]string, error) { - return getCSVReader(decode.in).ReadAll() -} - -func (decode *decoder) getCSVRow() ([]string, error) { - if decode.csvDecoder == nil { - decode.csvDecoder = &csvDecoder{getCSVReader(decode.in)} - } - return decode.csvDecoder.Read() + getCSVRows() ([][]string, error) } type CSVReader interface { @@ -47,6 +28,18 @@ type csvDecoder struct { CSVReader } +func newSimpleDecoderFromReader(r io.Reader) SimpleDecoder { + return csvDecoder{getCSVReader(r)} +} + +// NewSimpleDecoderFromCSVReader creates a SimpleDecoder, which may be passed +// to the UnmarshalDecoder* family of functions, from a CSV reader. Note that +// encoding/csv.Reader implements CSVReader, so you can pass one of those +// directly here. +func NewSimpleDecoderFromCSVReader(r CSVReader) SimpleDecoder { + return csvDecoder{r} +} + func (c csvDecoder) getCSVRows() ([][]string, error) { return c.ReadAll() } @@ -87,7 +80,7 @@ func mismatchHeaderFields(structInfo []fieldInfo, headers []string) []string { return missing } - keyMap := make(map[string]struct{}, 0) + keyMap := make(map[string]struct{}) for _, info := range structInfo { for _, key := range info.keys { keyMap[key] = struct{}{} @@ -115,14 +108,27 @@ func maybeDoubleHeaderNames(headers []string) error { headerMap := make(map[string]bool, len(headers)) for _, v := range headers { if _, ok := headerMap[v]; ok { - return fmt.Errorf("Repeated header name: %v", v) + return fmt.Errorf("repeated header name: %v", v) } headerMap[v] = true } return nil } +// apply normalizer func to headers +func normalizeHeaders(headers []string) []string { + out := make([]string, len(headers)) + for i, h := range headers { + out[i] = normalizeName(h) + } + return out +} + func readTo(decoder Decoder, out interface{}) error { + return readToWithErrorHandler(decoder, nil, out) +} + +func readToWithErrorHandler(decoder Decoder, errHandler ErrorHandler, out interface{}) error { outValue, outType := getConcreteReflectValueAndType(out) // Get the concrete type (not pointer) (Slice or Array) if err := ensureOutType(outType); err != nil { return err @@ -146,7 +152,7 @@ func readTo(decoder Decoder, out interface{}) error { return errors.New("no csv struct tags found") } - headers := csvRows[0] + headers := normalizeHeaders(csvRows[0]) body := csvRows[1:] csvHeadersLabels := make(map[int]*fieldInfo, len(outInnerStructInfo.Fields)) // Used to store the correspondance header <-> position in CSV @@ -174,34 +180,66 @@ func readTo(decoder Decoder, out interface{}) error { } } + var withFieldsOK bool + var fieldTypeUnmarshallerWithKeys TypeUnmarshalCSVWithFields + for i, csvRow := range body { + objectIface := reflect.New(outValue.Index(i).Type()).Interface() outInner := createNewOutInner(outInnerWasPointer, outInnerType) for j, csvColumnContent := range csvRow { if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name + + if outInner.CanInterface() { + fieldTypeUnmarshallerWithKeys, withFieldsOK = objectIface.(TypeUnmarshalCSVWithFields) + if withFieldsOK { + if err := fieldTypeUnmarshallerWithKeys.UnmarshalCSVWithFields(fieldInfo.getFirstKey(), csvColumnContent); err != nil { + parseError := csv.ParseError{ + Line: i + 2, //add 2 to account for the header & 0-indexing of arrays + Column: j + 1, + Err: err, + } + return &parseError + } + continue + } + } + if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct - return &csv.ParseError{ + parseError := csv.ParseError{ Line: i + 2, //add 2 to account for the header & 0-indexing of arrays Column: j + 1, Err: err, } + if errHandler == nil || !errHandler(&parseError) { + return &parseError + } } } } + + if withFieldsOK { + reflectedObject := reflect.ValueOf(objectIface) + outInner = reflectedObject.Elem() + } + outValue.Index(i).Set(outInner) } return nil } func readEach(decoder SimpleDecoder, c interface{}) error { + outValue, outType := getConcreteReflectValueAndType(c) // Get the concrete type (not pointer) + if outType.Kind() != reflect.Chan { + return fmt.Errorf("cannot use %v with type %s, only channel supported", c, outType) + } + defer outValue.Close() + headers, err := decoder.getCSVRow() if err != nil { return err } - outValue, outType := getConcreteReflectValueAndType(c) // Get the concrete type (not pointer) (Slice or Array) - if err := ensureOutType(outType); err != nil { - return err - } - defer outValue.Close() + headers = normalizeHeaders(headers) + outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">) if err := ensureOutInnerType(outInnerType); err != nil { return err @@ -258,6 +296,89 @@ func readEach(decoder SimpleDecoder, c interface{}) error { return nil } +func readEachWithoutHeaders(decoder SimpleDecoder, c interface{}) error { + outValue, outType := getConcreteReflectValueAndType(c) // Get the concrete type (not pointer) (Slice or Array) + if err := ensureOutType(outType); err != nil { + return err + } + defer outValue.Close() + + outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">) + if err := ensureOutInnerType(outInnerType); err != nil { + return err + } + outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations + if len(outInnerStructInfo.Fields) == 0 { + return errors.New("no csv struct tags found") + } + + i := 0 + for { + line, err := decoder.getCSVRow() + if err == io.EOF { + break + } else if err != nil { + return err + } + outInner := createNewOutInner(outInnerWasPointer, outInnerType) + for j, csvColumnContent := range line { + fieldInfo := outInnerStructInfo.Fields[j] + if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct + return &csv.ParseError{ + Line: i + 2, //add 2 to account for the header & 0-indexing of arrays + Column: j + 1, + Err: err, + } + } + } + outValue.Send(outInner) + i++ + } + return nil +} + +func readToWithoutHeaders(decoder Decoder, out interface{}) error { + outValue, outType := getConcreteReflectValueAndType(out) // Get the concrete type (not pointer) (Slice or Array) + if err := ensureOutType(outType); err != nil { + return err + } + outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">) + if err := ensureOutInnerType(outInnerType); err != nil { + return err + } + csvRows, err := decoder.getCSVRows() // Get the CSV csvRows + if err != nil { + return err + } + if len(csvRows) == 0 { + return errors.New("empty csv file given") + } + if err := ensureOutCapacity(&outValue, len(csvRows)+1); err != nil { // Ensure the container is big enough to hold the CSV content + return err + } + outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations + if len(outInnerStructInfo.Fields) == 0 { + return errors.New("no csv struct tags found") + } + + for i, csvRow := range csvRows { + outInner := createNewOutInner(outInnerWasPointer, outInnerType) + for j, csvColumnContent := range csvRow { + fieldInfo := outInnerStructInfo.Fields[j] + if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct + return &csv.ParseError{ + Line: i + 1, + Column: j + 1, + Err: err, + } + } + } + outValue.Index(i).Set(outInner) + } + + return nil +} + // Check if the outType is an array or a slice func ensureOutType(outType reflect.Type) error { switch outType.Kind() { @@ -302,9 +423,8 @@ func getCSVFieldPosition(key string, structInfo *structInfo, curHeaderCount int) if field.matchesKey(key) { if matchedFieldCount >= curHeaderCount { return &field - } else { - matchedFieldCount++ } + matchedFieldCount++ } } return nil @@ -320,7 +440,16 @@ func createNewOutInner(outInnerWasPointer bool, outInnerType reflect.Type) refle func setInnerField(outInner *reflect.Value, outInnerWasPointer bool, index []int, value string, omitEmpty bool) error { oi := *outInner if outInnerWasPointer { + // initialize nil pointer + if oi.IsNil() { + setField(oi, "", omitEmpty) + } oi = outInner.Elem() } + // because pointers can be nil need to recurse one index at a time and perform nil check + if len(index) > 1 { + nextField := oi.Field(index[0]) + return setInnerField(&nextField, nextField.Kind() == reflect.Ptr, index[1:], value, omitEmpty) + } return setField(oi.FieldByIndex(index), value, omitEmpty) } diff --git a/vendor/github.com/gocarina/gocsv/encode.go b/vendor/github.com/gocarina/gocsv/encode.go index 7a41f9f..8671533 100644 --- a/vendor/github.com/gocarina/gocsv/encode.go +++ b/vendor/github.com/gocarina/gocsv/encode.go @@ -133,7 +133,15 @@ func ensureInInnerType(outInnerType reflect.Type) error { func getInnerField(outInner reflect.Value, outInnerWasPointer bool, index []int) (string, error) { oi := outInner if outInnerWasPointer { + if oi.IsNil() { + return "", nil + } oi = outInner.Elem() } + // because pointers can be nil need to recurse one index at a time and perform nil check + if len(index) > 1 { + nextField := oi.Field(index[0]) + return getInnerField(nextField, nextField.Kind() == reflect.Ptr, index[1:]) + } return getFieldAsString(oi.FieldByIndex(index)) } diff --git a/vendor/github.com/gocarina/gocsv/go.mod b/vendor/github.com/gocarina/gocsv/go.mod new file mode 100644 index 0000000..c746a5a --- /dev/null +++ b/vendor/github.com/gocarina/gocsv/go.mod @@ -0,0 +1,3 @@ +module github.com/gocarina/gocsv + +go 1.13 diff --git a/vendor/github.com/gocarina/gocsv/reflect.go b/vendor/github.com/gocarina/gocsv/reflect.go index 9217e30..dfc63b3 100644 --- a/vendor/github.com/gocarina/gocsv/reflect.go +++ b/vendor/github.com/gocarina/gocsv/reflect.go @@ -35,19 +35,21 @@ func (f fieldInfo) matchesKey(key string) bool { return false } +var structInfoCache sync.Map var structMap = make(map[reflect.Type]*structInfo) var structMapMutex sync.RWMutex func getStructInfo(rType reflect.Type) *structInfo { - structMapMutex.RLock() - stInfo, ok := structMap[rType] - structMapMutex.RUnlock() + stInfo, ok := structInfoCache.Load(rType) if ok { - return stInfo + return stInfo.(*structInfo) } + fieldsList := getFieldInfos(rType, []int{}) stInfo = &structInfo{fieldsList} - return stInfo + structInfoCache.Store(rType, stInfo) + + return stInfo.(*structInfo) } func getFieldInfos(rType reflect.Type, parentIndexChain []int) []fieldInfo { @@ -58,19 +60,40 @@ func getFieldInfos(rType reflect.Type, parentIndexChain []int) []fieldInfo { if field.PkgPath != "" { continue } - indexChain := append(parentIndexChain, i) - // if the field is an embedded struct, create a fieldInfo for each of its fields - if field.Anonymous && field.Type.Kind() == reflect.Struct { - fieldsList = append(fieldsList, getFieldInfos(field.Type, indexChain)...) + + var cpy = make([]int, len(parentIndexChain)) + copy(cpy, parentIndexChain) + indexChain := append(cpy, i) + + // if the field is a pointer to a struct, follow the pointer then create fieldinfo for each field + if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct { + // unless it implements marshalText or marshalCSV. Structs that implement this + // should result in one value and not have their fields exposed + if !(canMarshal(field.Type.Elem())) { + fieldsList = append(fieldsList, getFieldInfos(field.Type.Elem(), indexChain)...) + } + } + // if the field is a struct, create a fieldInfo for each of its fields + if field.Type.Kind() == reflect.Struct { + // unless it implements marshalText or marshalCSV. Structs that implement this + // should result in one value and not have their fields exposed + if !(canMarshal(field.Type)) { + fieldsList = append(fieldsList, getFieldInfos(field.Type, indexChain)...) + } + } + + // if the field is an embedded struct, ignore the csv tag + if field.Anonymous { continue } + fieldInfo := fieldInfo{IndexChain: indexChain} - fieldTag := field.Tag.Get("csv") + fieldTag := field.Tag.Get(TagName) fieldTags := strings.Split(fieldTag, TagSeparator) filteredTags := []string{} for _, fieldTagEntry := range fieldTags { if fieldTagEntry != "omitempty" { - filteredTags = append(filteredTags, fieldTagEntry) + filteredTags = append(filteredTags, normalizeName(fieldTagEntry)) } else { fieldInfo.omitEmpty = true } @@ -81,7 +104,7 @@ func getFieldInfos(rType reflect.Type, parentIndexChain []int) []fieldInfo { } else if len(filteredTags) > 0 && filteredTags[0] != "" { fieldInfo.keys = filteredTags } else { - fieldInfo.keys = []string{field.Name} + fieldInfo.keys = []string{normalizeName(field.Name)} } fieldsList = append(fieldsList, fieldInfo) } @@ -105,3 +128,13 @@ func getConcreteReflectValueAndType(in interface{}) (reflect.Value, reflect.Type } return value, value.Type() } + +var errorInterface = reflect.TypeOf((*error)(nil)).Elem() + +func isErrorType(outType reflect.Type) bool { + if outType.Kind() != reflect.Interface { + return false + } + + return outType.Implements(errorInterface) +} diff --git a/vendor/github.com/gocarina/gocsv/types.go b/vendor/github.com/gocarina/gocsv/types.go index 50d88ce..a5dda89 100644 --- a/vendor/github.com/gocarina/gocsv/types.go +++ b/vendor/github.com/gocarina/gocsv/types.go @@ -6,6 +6,8 @@ import ( "reflect" "strconv" "strings" + + "encoding/json" ) // -------------------------------------------------------------------------- @@ -17,19 +19,17 @@ type TypeMarshaller interface { MarshalCSV() (string, error) } -// Stringer is implemented by any value that has a String method -// This converter is used to convert the value to it string representation -// This converter will be used if your value does not implement TypeMarshaller -type Stringer interface { - String() string -} - // TypeUnmarshaller is implemented by any value that has an UnmarshalCSV method // This converter is used to convert a string to your value representation of that string type TypeUnmarshaller interface { UnmarshalCSV(string) error } +// TypeUnmarshalCSVWithFields can be implemented on whole structs to allow for whole structures to customized internal vs one off fields +type TypeUnmarshalCSVWithFields interface { + UnmarshalCSVWithFields(key, value string) error +} + // NoUnmarshalFuncError is the custom error type to be raised in case there is no unmarshal function defined on type type NoUnmarshalFuncError struct { msg string @@ -41,21 +41,13 @@ func (e NoUnmarshalFuncError) Error() string { // NoMarshalFuncError is the custom error type to be raised in case there is no marshal function defined on type type NoMarshalFuncError struct { - msg string + ty reflect.Type } func (e NoMarshalFuncError) Error() string { - return e.msg + return "No known conversion from " + e.ty.String() + " to string, " + e.ty.String() + " does not implement TypeMarshaller nor Stringer" } -var ( - stringerType = reflect.TypeOf((*Stringer)(nil)).Elem() - marshallerType = reflect.TypeOf((*TypeMarshaller)(nil)).Elem() - unMarshallerType = reflect.TypeOf((*TypeUnmarshaller)(nil)).Elem() - textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() - textUnMarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() -) - // -------------------------------------------------------------------------- // Conversion helpers @@ -89,12 +81,12 @@ func toBool(in interface{}) (bool, error) { switch inValue.Kind() { case reflect.String: s := inValue.String() - switch s { - case "yes": + s = strings.TrimSpace(s) + if strings.EqualFold(s, "yes") { return true, nil - case "no", "": + } else if strings.EqualFold(s, "no") || s == "" { return false, nil - default: + } else { return strconv.ParseBool(s) } case reflect.Bool: @@ -130,7 +122,8 @@ func toInt(in interface{}) (int64, error) { if s == "" { return 0, nil } - return strconv.ParseInt(s, 0, 64) + out := strings.SplitN(s, ".", 2) + return strconv.ParseInt(out[0], 0, 64) case reflect.Bool: if inValue.Bool() { return 1, nil @@ -285,6 +278,11 @@ func setField(field reflect.Value, value string, omitEmpty bool) error { return err } field.SetFloat(f) + case reflect.Slice, reflect.Struct: + err := json.Unmarshal([]byte(value), field.Addr().Interface()) + if err != nil { + return err + } default: return err } @@ -297,32 +295,28 @@ func setField(field reflect.Value, value string, omitEmpty bool) error { func getFieldAsString(field reflect.Value) (str string, err error) { switch field.Kind() { - case reflect.Interface: - case reflect.Ptr: + case reflect.Interface, reflect.Ptr: if field.IsNil() { return "", nil } return getFieldAsString(field.Elem()) + case reflect.String: + return field.String(), nil default: // Check if field is go native type switch field.Interface().(type) { case string: return field.String(), nil case bool: - str, err = toString(field.Bool()) - if err != nil { - return str, err + if field.Bool() { + return "true", nil + } else { + return "false", nil } case int, int8, int16, int32, int64: - str, err = toString(field.Int()) - if err != nil { - return str, err - } + return fmt.Sprintf("%v", field.Int()), nil case uint, uint8, uint16, uint32, uint64: - str, err = toString(field.Uint()) - if err != nil { - return str, err - } + return fmt.Sprintf("%v", field.Uint()), nil case float32: str, err = toString(float32(field.Float())) if err != nil { @@ -381,6 +375,14 @@ func getFieldAsString(field reflect.Value) (str string, err error) { // -------------------------------------------------------------------------- // Un/serializations helpers +func canMarshal(t reflect.Type) bool { + // unless it implements marshalText or marshalCSV. Structs that implement this + // should result in one value and not have their fields exposed + _, canMarshalText := t.MethodByName("MarshalText") + _, canMarshalCSV := t.MethodByName("MarshalCSV") + return canMarshalCSV || canMarshalText +} + func unmarshall(field reflect.Value, value string) error { dupField := field unMarshallIt := func(finalField reflect.Value) error { @@ -435,13 +437,13 @@ func marshall(field reflect.Value) (value string, err error) { } // Otherwise try to use Stringer - fieldStringer, ok := fieldIface.(Stringer) + fieldStringer, ok := fieldIface.(fmt.Stringer) if ok { return fieldStringer.String(), nil } } - return value, NoMarshalFuncError{"No known conversion from " + field.Type().String() + " to string, " + field.Type().String() + " does not implement TypeMarshaller nor Stringer"} + return value, NoMarshalFuncError{field.Type()} } for dupField.Kind() == reflect.Interface || dupField.Kind() == reflect.Ptr { if dupField.IsNil() { @@ -450,7 +452,7 @@ func marshall(field reflect.Value) (value string, err error) { dupField = dupField.Elem() } if dupField.CanAddr() { - return marshallIt(dupField.Addr()) + dupField = dupField.Addr() } - return value, NoMarshalFuncError{"No known conversion from " + field.Type().String() + " to string, " + field.Type().String() + " does not implement TypeMarshaller nor Stringer"} + return marshallIt(dupField) } diff --git a/vendor/github.com/gocarina/gocsv/unmarshaller.go b/vendor/github.com/gocarina/gocsv/unmarshaller.go index ace601e..87e6a8f 100644 --- a/vendor/github.com/gocarina/gocsv/unmarshaller.go +++ b/vendor/github.com/gocarina/gocsv/unmarshaller.go @@ -10,7 +10,8 @@ import ( // Unmarshaller is a CSV to struct unmarshaller. type Unmarshaller struct { reader *csv.Reader - fieldInfoMap map[int]*fieldInfo + Headers []string + fieldInfoMap []*fieldInfo MismatchedHeaders []string MismatchedStructFields []string outType reflect.Type @@ -22,6 +23,7 @@ func NewUnmarshaller(reader *csv.Reader, out interface{}) (*Unmarshaller, error) if err != nil { return nil, err } + headers = normalizeHeaders(headers) um := &Unmarshaller{reader: reader, outType: reflect.TypeOf(out)} err = validate(um, out, headers) @@ -38,7 +40,18 @@ func (um *Unmarshaller) Read() (interface{}, error) { if err != nil { return nil, err } - return um.unmarshalRow(row) + return um.unmarshalRow(row, nil) +} + +// ReadUnmatched is same as Read(), but returns a map of the columns that didn't match a field in the struct +func (um *Unmarshaller) ReadUnmatched() (interface{}, map[string]string, error) { + row, err := um.reader.Read() + if err != nil { + return nil, nil, err + } + unmatched := make(map[string]string) + value, err := um.unmarshalRow(row, unmatched) + return value, unmatched, err } // validate ensures that a struct was used to create the Unmarshaller, and validates @@ -55,7 +68,7 @@ func validate(um *Unmarshaller, s interface{}, headers []string) error { if len(structInfo.Fields) == 0 { return errors.New("no csv struct tags found") } - csvHeadersLabels := make(map[int]*fieldInfo, len(structInfo.Fields)) // Used to store the corresponding header <-> position in CSV + csvHeadersLabels := make([]*fieldInfo, len(headers)) // Used to store the corresponding header <-> position in CSV headerCount := map[string]int{} for i, csvColumnHeader := range headers { curHeaderCount := headerCount[csvColumnHeader] @@ -67,10 +80,14 @@ func validate(um *Unmarshaller, s interface{}, headers []string) error { } } } - if err := maybeDoubleHeaderNames(headers); err != nil { - return err + + if FailIfDoubleHeaderNames { + if err := maybeDoubleHeaderNames(headers); err != nil { + return err + } } + um.Headers = headers um.fieldInfoMap = csvHeadersLabels um.MismatchedHeaders = mismatchHeaderFields(structInfo.Fields, headers) um.MismatchedStructFields = mismatchStructFields(structInfo.Fields, headers) @@ -78,7 +95,8 @@ func validate(um *Unmarshaller, s interface{}, headers []string) error { } // unmarshalRow converts a CSV row to a struct, based on CSV struct tags. -func (um *Unmarshaller) unmarshalRow(row []string) (interface{}, error) { +// If unmatched is non nil, it is populated with any columns that don't map to a struct field +func (um *Unmarshaller) unmarshalRow(row []string, unmatched map[string]string) (interface{}, error) { isPointer := false concreteOutType := um.outType if um.outType.Kind() == reflect.Ptr { @@ -87,10 +105,13 @@ func (um *Unmarshaller) unmarshalRow(row []string) (interface{}, error) { } outValue := createNewOutInner(isPointer, concreteOutType) for j, csvColumnContent := range row { - if fieldInfo, ok := um.fieldInfoMap[j]; ok { + if j < len(um.fieldInfoMap) && um.fieldInfoMap[j] != nil { + fieldInfo := um.fieldInfoMap[j] if err := setInnerField(&outValue, isPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct return nil, fmt.Errorf("cannot assign field at %v to %s through index chain %v: %v", j, outValue.Type(), fieldInfo.IndexChain, err) } + } else if unmatched != nil { + unmatched[um.Headers[j]] = csvColumnContent } } return outValue.Interface(), nil -- cgit v1.2.3