Skip to content

Commit

Permalink
feat: init es reader and writer
Browse files Browse the repository at this point in the history
  • Loading branch information
Breeze0806 committed Feb 17, 2025
1 parent 009d884 commit 3a7a456
Show file tree
Hide file tree
Showing 14 changed files with 95 additions and 1 deletion.
2 changes: 1 addition & 1 deletion .github/workflows/Build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches:
- main
- v0.2.x
- v0.2.x-es
pull_request:
branches:
- main
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ This data synchronization tool has the synchronization capability for the follow
| | Sqlite3 ||| [Read](datax/plugin/reader/sqlite3/README.md)[Write](datax/plugin/writer/sqlite3/README.md) |
| Unstructured Data Stream | CSV ||| [Read](datax/plugin/reader/csv/README.md)[Write](datax/plugin/writer/csv/README.md) |
| | XLSX(excel) ||| [Read](datax/plugin/reader/xlsx/README.md)[Write](datax/plugin/writer/xlsx/README.md) |
| | Elasticsearch ||| [Read](datax/plugin/reader/elasticsearch/README.md)[Write](datax/plugin/writer/elasticsearch/README.md) |


### Getting Started

Expand Down
1 change: 1 addition & 0 deletions datax/plugin/reader/elasticsearch/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# ElasticsearchReader Plugin Documentation
1 change: 1 addition & 0 deletions datax/plugin/reader/elasticsearch/job.go
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package elasticsearch
31 changes: 31 additions & 0 deletions datax/plugin/reader/elasticsearch/reader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package elasticsearch
import (
"github.com/Breeze0806/go-etl/config"
spireader "github.com/Breeze0806/go-etl/datax/common/spi/reader"
)

//A reader is uesed to extract data from data source
type Reader struct {
pluginConf *config.JSON
}

//ResourcesConfig returns the configuration of the data source to initiate the reader.
func (r *Reader) ResourcesConfig() *config.JSON {
return r.pluginConf
}

//Job returns a description of how the reader extracts data from the data source.
func (r *Reader) Job() spireader.Job {
// todo like below
//job := NewJob()
//job.SetPluginConf(r.pluginConf)
//return job
}

//Task returns the smallest execution unit obtained by maximizing the split of a Job
func (r *Reader) Task() spireader.Task {
// todo like below
//task := fNewTask()
//task.SetPluginConf(r.pluginConf)
//return task
}
5 changes: 5 additions & 0 deletions datax/plugin/reader/elasticsearch/resources/plugin.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"name" : "elasticsearchreader",
"developer":"",
"description":""
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"name": "elasticsearchreader",
"parameter": {

}
}
1 change: 1 addition & 0 deletions datax/plugin/reader/elasticsearch/task.go
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package elasticsearch
1 change: 1 addition & 0 deletions datax/plugin/writer/elasticsearch/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# ElasticsearchWriter Plugin Documentation
1 change: 1 addition & 0 deletions datax/plugin/writer/elasticsearch/job.go
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package elasticsearch
5 changes: 5 additions & 0 deletions datax/plugin/writer/elasticsearch/resources/plugin.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"name" : "elasticsearchwriter",
"developer":"",
"description":""
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"name": "elasticsearchwriter",
"parameter": {

}
}
1 change: 1 addition & 0 deletions datax/plugin/writer/elasticsearch/task.go
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package elasticsearch
33 changes: 33 additions & 0 deletions datax/plugin/writer/elasticsearch/writer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package elasticsearch

import (
"github.com/Breeze0806/go-etl/config"
spiwriter "github.com/Breeze0806/go-etl/datax/common/spi/writer"
)


//A writer is uesed to load data into data source
type Writer struct {
pluginConf *config.JSON
}

//ResourcesConfig returns the configuration of the data source to initiate the writer.
func (w *Writer) ResourcesConfig() *config.JSON {
return w.pluginConf
}

//Job returns a description of how the reader extracts data from the data source.
func (w *Writer) Job() spiwriter.Job {
// todo like below
//job := NewJob()
//job.SetPluginConf(w.pluginConf)
//return job
}

//Task returns the smallest execution unit obtained by maximizing the split of a Job
func (w *Writer) Task() spiwriter.Task {
// todo like below
//task := NewTask()
//task.SetPluginConf(w.pluginConf)
//return task
}

0 comments on commit 3a7a456

Please sign in to comment.