From 3a7a45612cbc2671dcd6b077777f20e3b825d7eb Mon Sep 17 00:00:00 2001 From: Breeze0806 Date: Mon, 17 Feb 2025 20:05:47 +0800 Subject: [PATCH] feat: init es reader and writer --- .github/workflows/Build.yml | 2 +- README.md | 2 ++ datax/plugin/reader/elasticsearch/README.md | 1 + datax/plugin/reader/elasticsearch/job.go | 1 + datax/plugin/reader/elasticsearch/reader.go | 31 +++++++++++++++++ .../elasticsearch/resources/plugin.json | 5 +++ .../resources/plugin_job_template.json | 6 ++++ datax/plugin/reader/elasticsearch/task.go | 1 + datax/plugin/writer/elasticsearch/README.md | 1 + datax/plugin/writer/elasticsearch/job.go | 1 + .../elasticsearch/resources/plugin.json | 5 +++ .../resources/plugin_job_template.json | 6 ++++ datax/plugin/writer/elasticsearch/task.go | 1 + datax/plugin/writer/elasticsearch/writer.go | 33 +++++++++++++++++++ 14 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 datax/plugin/reader/elasticsearch/README.md create mode 100644 datax/plugin/reader/elasticsearch/job.go create mode 100644 datax/plugin/reader/elasticsearch/reader.go create mode 100644 datax/plugin/reader/elasticsearch/resources/plugin.json create mode 100644 datax/plugin/reader/elasticsearch/resources/plugin_job_template.json create mode 100644 datax/plugin/reader/elasticsearch/task.go create mode 100644 datax/plugin/writer/elasticsearch/README.md create mode 100644 datax/plugin/writer/elasticsearch/job.go create mode 100644 datax/plugin/writer/elasticsearch/resources/plugin.json create mode 100644 datax/plugin/writer/elasticsearch/resources/plugin_job_template.json create mode 100644 datax/plugin/writer/elasticsearch/task.go create mode 100644 datax/plugin/writer/elasticsearch/writer.go diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 98098d2..783776c 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -4,7 +4,7 @@ on: push: branches: - main - - v0.2.x + - v0.2.x-es pull_request: branches: - main diff --git a/README.md b/README.md index f0a4df1..8055a93 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ This data synchronization tool has the synchronization capability for the follow | | Sqlite3 | √ | √ | [Read](datax/plugin/reader/sqlite3/README.md)、[Write](datax/plugin/writer/sqlite3/README.md) | | Unstructured Data Stream | CSV | √ | √ | [Read](datax/plugin/reader/csv/README.md)、[Write](datax/plugin/writer/csv/README.md) | | | XLSX（excel） | √ | √ | [Read](datax/plugin/reader/xlsx/README.md)、[Write](datax/plugin/writer/xlsx/README.md) | +| | Elasticsearch | √ | √ | [Read](datax/plugin/reader/elasticsearch/README.md)、[Write](datax/plugin/writer/elasticsearch/README.md) | + ### Getting Started diff --git a/datax/plugin/reader/elasticsearch/README.md b/datax/plugin/reader/elasticsearch/README.md new file mode 100644 index 0000000..0f2cbd8 --- /dev/null +++ b/datax/plugin/reader/elasticsearch/README.md @@ -0,0 +1 @@ +# ElasticsearchReader Plugin Documentation diff --git a/datax/plugin/reader/elasticsearch/job.go b/datax/plugin/reader/elasticsearch/job.go new file mode 100644 index 0000000..c2206bc --- /dev/null +++ b/datax/plugin/reader/elasticsearch/job.go @@ -0,0 +1 @@ +package elasticsearch diff --git a/datax/plugin/reader/elasticsearch/reader.go b/datax/plugin/reader/elasticsearch/reader.go new file mode 100644 index 0000000..2ff4b3f --- /dev/null +++ b/datax/plugin/reader/elasticsearch/reader.go @@ -0,0 +1,31 @@ +package elasticsearch +import ( + "github.com/Breeze0806/go-etl/config" + spireader "github.com/Breeze0806/go-etl/datax/common/spi/reader" +) + +//A reader is uesed to extract data from data source +type Reader struct { + pluginConf *config.JSON +} + +//ResourcesConfig returns the configuration of the data source to initiate the reader. +func (r *Reader) ResourcesConfig() *config.JSON { + return r.pluginConf +} + +//Job returns a description of how the reader extracts data from the data source. +func (r *Reader) Job() spireader.Job { + // todo like below + //job := NewJob() + //job.SetPluginConf(r.pluginConf) + //return job +} + +//Task returns the smallest execution unit obtained by maximizing the split of a Job +func (r *Reader) Task() spireader.Task { + // todo like below + //task := fNewTask() + //task.SetPluginConf(r.pluginConf) + //return task +} diff --git a/datax/plugin/reader/elasticsearch/resources/plugin.json b/datax/plugin/reader/elasticsearch/resources/plugin.json new file mode 100644 index 0000000..493bcf5 --- /dev/null +++ b/datax/plugin/reader/elasticsearch/resources/plugin.json @@ -0,0 +1,5 @@ +{ + "name" : "elasticsearchreader", + "developer":"", + "description":"" +} \ No newline at end of file diff --git a/datax/plugin/reader/elasticsearch/resources/plugin_job_template.json b/datax/plugin/reader/elasticsearch/resources/plugin_job_template.json new file mode 100644 index 0000000..5b2a601 --- /dev/null +++ b/datax/plugin/reader/elasticsearch/resources/plugin_job_template.json @@ -0,0 +1,6 @@ +{ + "name": "elasticsearchreader", + "parameter": { + + } +} \ No newline at end of file diff --git a/datax/plugin/reader/elasticsearch/task.go b/datax/plugin/reader/elasticsearch/task.go new file mode 100644 index 0000000..c2206bc --- /dev/null +++ b/datax/plugin/reader/elasticsearch/task.go @@ -0,0 +1 @@ +package elasticsearch diff --git a/datax/plugin/writer/elasticsearch/README.md b/datax/plugin/writer/elasticsearch/README.md new file mode 100644 index 0000000..05cb069 --- /dev/null +++ b/datax/plugin/writer/elasticsearch/README.md @@ -0,0 +1 @@ +# ElasticsearchWriter Plugin Documentation diff --git a/datax/plugin/writer/elasticsearch/job.go b/datax/plugin/writer/elasticsearch/job.go new file mode 100644 index 0000000..c2206bc --- /dev/null +++ b/datax/plugin/writer/elasticsearch/job.go @@ -0,0 +1 @@ +package elasticsearch diff --git a/datax/plugin/writer/elasticsearch/resources/plugin.json b/datax/plugin/writer/elasticsearch/resources/plugin.json new file mode 100644 index 0000000..4b59e0b --- /dev/null +++ b/datax/plugin/writer/elasticsearch/resources/plugin.json @@ -0,0 +1,5 @@ +{ + "name" : "elasticsearchwriter", + "developer":"", + "description":"" +} \ No newline at end of file diff --git a/datax/plugin/writer/elasticsearch/resources/plugin_job_template.json b/datax/plugin/writer/elasticsearch/resources/plugin_job_template.json new file mode 100644 index 0000000..ba1159c --- /dev/null +++ b/datax/plugin/writer/elasticsearch/resources/plugin_job_template.json @@ -0,0 +1,6 @@ +{ + "name": "elasticsearchwriter", + "parameter": { + + } +} \ No newline at end of file diff --git a/datax/plugin/writer/elasticsearch/task.go b/datax/plugin/writer/elasticsearch/task.go new file mode 100644 index 0000000..c2206bc --- /dev/null +++ b/datax/plugin/writer/elasticsearch/task.go @@ -0,0 +1 @@ +package elasticsearch diff --git a/datax/plugin/writer/elasticsearch/writer.go b/datax/plugin/writer/elasticsearch/writer.go new file mode 100644 index 0000000..72d91a3 --- /dev/null +++ b/datax/plugin/writer/elasticsearch/writer.go @@ -0,0 +1,33 @@ +package elasticsearch + +import ( + "github.com/Breeze0806/go-etl/config" + spiwriter "github.com/Breeze0806/go-etl/datax/common/spi/writer" +) + + +//A writer is uesed to load data into data source +type Writer struct { + pluginConf *config.JSON +} + +//ResourcesConfig returns the configuration of the data source to initiate the writer. +func (w *Writer) ResourcesConfig() *config.JSON { + return w.pluginConf +} + +//Job returns a description of how the reader extracts data from the data source. +func (w *Writer) Job() spiwriter.Job { + // todo like below + //job := NewJob() + //job.SetPluginConf(w.pluginConf) + //return job +} + +//Task returns the smallest execution unit obtained by maximizing the split of a Job +func (w *Writer) Task() spiwriter.Task { + // todo like below + //task := NewTask() + //task.SetPluginConf(w.pluginConf) + //return task +}