- Address issue: #62 (Chinese characters and pythong BOM prefix)

rocketlaunchr · Apr 2, 2022 · 418f042 · 418f042
1 parent a103044
commit 418f042
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/exports/parquet.go b/exports/parquet.go
@@ -65,7 +65,7 @@ func ExportToParquet(ctx context.Context, w io.Writer, df *dataframe.DataFrame,
  // Create Schema
  dataSchema := dynamicstruct.NewStruct()
  for _, aSeries := range df.Series {
- fieldName := strings.Title(strings.ToLower(aSeries.Name()))
+ fieldName := "Z" + strings.Title(strings.ToLower(aSeries.Name())) // Make it validly exported
  seriesName := santizeColumnName(aSeries.Name())
 
  switch aSeries.(type) {
@@ -125,7 +125,7 @@ func ExportToParquet(ctx context.Context, w io.Writer, df *dataframe.DataFrame,
 
  rec := schemaStruct.New()
  for _, aSeries := range df.Series {
- fieldName := strings.Title(strings.ToLower(aSeries.Name()))
+ fieldName := "Z" + strings.Title(strings.ToLower(aSeries.Name()))
 
  v := reflect.ValueOf(rec).Elem().FieldByName(fieldName)
  if v.IsValid() {

diff --git a/imports/csv.go b/imports/csv.go
@@ -74,6 +74,22 @@ func LoadFromCSV(ctx context.Context, r io.ReadSeeker, options ...CSVLoadOptions
 
  var init *dataframe.SeriesInit
 
+ // Check for bom characters in the beginning (that python seems to add).
+ // See:
+ // https://github.com/rocketlaunchr/dataframe-go/issues/62
+ // https://github.com/golang/go/issues/33887
+ // https://github.com/dimchansky/utfbom
+ // https://github.com/spkg/bom/
+ checkBOM := make([]byte, 3)
+ readN, err := r.Read(checkBOM)
+ if err != nil {
+ return nil, err
+ }
+ if !(readN == 3 && checkBOM[0] == 0xef && checkBOM[1] == 0xbb && checkBOM[2] == 0xbf) {
+ // bom not found so reset reader
+ r.Seek(0, io.SeekStart)
+ }
+
  var (
  comma rune
  comment rune