vesoft-inc · yixinglu · Jan 10, 2024 · Nov 20, 2023
diff --git a/examples/basic/basic.int.delete.v3.yaml b/examples/basic/basic.int.delete.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: DELETE

diff --git a/examples/basic/basic.int.insert.v3.yaml b/examples/basic/basic.int.insert.v3.yaml
@@ -35,6 +35,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  id:

diff --git a/examples/basic/basic.int.update.v3.yaml b/examples/basic/basic.int.update.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: UPDATE

diff --git a/examples/basic/basic.string.delete.v3.yaml b/examples/basic/basic.string.delete.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: DELETE

diff --git a/examples/basic/basic.string.insert.v3.yaml b/examples/basic/basic.string.insert.v3.yaml
@@ -35,6 +35,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  id:

diff --git a/examples/basic/basic.string.update.v3.yaml b/examples/basic/basic.string.update.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: UPDATE

diff --git a/examples/basic/person.csv b/examples/basic/person.csv
@@ -1,4 +1,7 @@
+# I'm comment
+# I'm comment
 933|Mahinda|Perera|male|1989-12-03|2010-02-14T15:32:10|119.235.7.103|Firefox
+# I'm comment
 1129|Carmen|Lepland|female|1984-02-18|2010-01-28T06:39:58|195.20.151.175|Internet Explorer
 2199023256684|A.|Rao|female|1985-08-02|2010-04-23T22:52:26|49.202.188.25|Firefox
 4398046512167|Gustavo|Arbelaez|male|1986-11-02|2010-06-16T20:53:47|190.96.189.165|Chrome

diff --git a/integration-testing/testdata/basic/basic.int.delete.v3.yaml b/integration-testing/testdata/basic/basic.int.delete.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: DELETE

diff --git a/integration-testing/testdata/basic/basic.int.insert.v3.yaml b/integration-testing/testdata/basic/basic.int.insert.v3.yaml
@@ -35,6 +35,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  id:

diff --git a/integration-testing/testdata/basic/basic.int.update.v3.yaml b/integration-testing/testdata/basic/basic.int.update.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: UPDATE

diff --git a/integration-testing/testdata/basic/basic.string.delete.v3.yaml b/integration-testing/testdata/basic/basic.string.delete.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: DELETE

diff --git a/integration-testing/testdata/basic/basic.string.insert.v3.yaml b/integration-testing/testdata/basic/basic.string.insert.v3.yaml
@@ -35,6 +35,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  id:

diff --git a/integration-testing/testdata/basic/basic.string.update.v3.yaml b/integration-testing/testdata/basic/basic.string.update.v3.yaml
@@ -25,6 +25,7 @@ sources:
  - path: ./person.csv
  csv:
  delimiter: "|"
+ comment: "#"
  tags:
  - name: Person
  mode: UPDATE

diff --git a/integration-testing/testdata/basic/person.csv b/integration-testing/testdata/basic/person.csv
@@ -1,4 +1,7 @@
+# I'm comment
+# I'm comment
 933|Mahinda|Perera|male|1989-12-03|2010-02-14T15:32:10|119.235.7.103|Firefox
+# I'm comment
 1129|Carmen|Lepland|female|1984-02-18|2010-01-28T06:39:58|195.20.151.175|Internet Explorer
 2199023256684|A.|Rao|female|1985-08-02|2010-04-23T22:52:26|49.202.188.25|Firefox
 4398046512167|Gustavo|Arbelaez|male|1986-11-02|2010-06-16T20:53:47|190.96.189.165|Chrome

diff --git a/pkg/reader/csv.go b/pkg/reader/csv.go
@@ -40,6 +40,9 @@ func NewCSVReader(s source.Source) RecordReader {
  if chars := []rune(c.CSV.Delimiter); len(chars) > 0 {
  cr.Comma = chars[0]
  }
+ if chars := []rune(c.CSV.Comment); len(chars) > 0 {
+ cr.Comment = chars[0]
+ }
  cr.LazyQuotes = c.CSV.LazyQuotes
 
  h.withHeader = c.CSV.WithHeader

diff --git a/pkg/reader/csv_test.go b/pkg/reader/csv_test.go
@@ -81,6 +81,7 @@ var _ = Describe("csvReader", func() {
  },
  CSV: &source.CSVConfig{
  Delimiter: "|",
+ Comment: "#",
  },
  })
  Expect(err).NotTo(HaveOccurred())
@@ -103,32 +104,32 @@ var _ = Describe("csvReader", func() {
  r := NewCSVReader(s)
  nBytes, err = r.Size()
  Expect(err).NotTo(HaveOccurred())
- Expect(nBytes).To(Equal(int64(33)))
+ Expect(nBytes).To(Equal(int64(117)))
 
  n, record, err = r.Read()
  Expect(err).NotTo(HaveOccurred())
- Expect(n).To(Equal(6))
+ Expect(n).To(Equal(20))
  Expect(record).To(Equal(spec.Record{"1", "2", "3"}))
 
  n, record, err = r.Read()
  Expect(err).NotTo(HaveOccurred())
- Expect(n).To(Equal(7))
+ Expect(n).To(Equal(35))
  Expect(record).To(Equal(spec.Record{"4", " 5", "6"}))
 
  n, record, err = r.Read()
  Expect(err).NotTo(HaveOccurred())
- Expect(n).To(Equal(8))
+ Expect(n).To(Equal(22))
  Expect(record).To(Equal(spec.Record{" 7", "8", " 9"}))
 
  n, record, err = r.Read()
  Expect(err).NotTo(HaveOccurred())
- Expect(n).To(Equal(12))
+ Expect(n).To(Equal(26))
  Expect(record).To(Equal(spec.Record{"10", " 11 ", " 12"}))
 
  n, record, err = r.Read()
  Expect(err).To(HaveOccurred())
  Expect(stderrors.Is(err, io.EOF)).To(BeTrue())
- Expect(n).To(Equal(0))
+ Expect(n).To(Equal(14))
  Expect(record).To(BeEmpty())
  })
  })

diff --git a/pkg/reader/testdata/local_delimiter.csv b/pkg/reader/testdata/local_delimiter.csv
@@ -1,4 +1,10 @@
+# I'm comment
 1|2|3
+# I'm comment
+# I'm comment
 4| 5|6
+# I'm comment
  7|8| 9
+# I'm comment
 10| 11 | 12
+# I'm comment
diff --git a/pkg/source/config.go b/pkg/source/config.go
@@ -15,6 +15,7 @@ type (
 
  CSVConfig struct {
  Delimiter string `yaml:"delimiter,omitempty"`
+ Comment string `yaml:"comment,omitempty"`
  WithHeader bool `yaml:"withHeader,omitempty"`
  LazyQuotes bool `yaml:"lazyQuotes,omitempty"`
  }