How to use REST API input to Elasticsearch via logstash

1. Install logstash-filter-rest

2.  In your logstash config file add filter

Use Logstash-input-jdbc to synchronize sqlserver data to elasticsearch

1. Installation and download mssql-jdbc-6.2.2.jre8.jar

bin/plugin install logstash-input-jdbc

go to Microsoft web site download mssql jdbc ㊣ , and then unzip gz file to /opt/logstash

2. Configuration

input {
    stdin {
    }
    jdbc {
     jdbc_driver_library => "/opt/logstash/sqljdbc_6.2/enu/mssql-jdbc-6.2.2.jre8.jar"
            jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
            jdbc_connection_string => "jdbc:sqlserver://127.0.0.1:1433;DatabaseName=test"
            jdbc_user => "sa"
            jdbc_password => "123456"
            # schedule => m H D M Y  
            # schedule => * 22  *  *  *     //will execute at 22:00 every day
            schedule => "* * * * *"
            jdbc_paging_enabled => true
            jdbc_page_size => 1000
            clean_run => false
            use_column_value => true            #設定查詢條件的欄位
            tracking_column => pk_id
            record_last_run => true
            last_run_metadata_path => "/opt/data/station_parameter.log"            #設定列名小寫
            lowercase_column_names => true
            statement_filepath => "/jdbcconfig/statement.sql"
            #索引的型別
            type => "jdbc"
    }
}

filter {
    json {
        source => "message"
        remove_field => ["message"]
    }
}

output {
    elasticsearch {
         hosts => ["elasticsearch:9200"]
        action => "index" 
        #set index name
        index => "test"
        document_id => "%{pk_id}"
    }
    stdout {
        #codec => json_lines
        #set output format
        codec => line {
            format => "pk_id: %{[pk_id]} name: %{[name]} lastedittime: %{[lastedittime]} " 
        }
    }
}

Run Test

bin/logstash -f config/jdbc.conf

Reference : logstash-input-jdbc同步mysql數據到elasticsearch
Reference : 利用 Logstash-input-jdbc同步sqlserver資料到elasticsearch
Reference : Logstash JDBC Input Plugin

Logstash REST Filter

1. Installation

You can use the built-in plugin tool of Logstash to install the filter:

bin/logstash-plugin install logstash-filter-rest

2. Filter Configuration

Add the following inside the filter section of your logstash configuration:

filter {
  rest {
    request => {
      url => "http://example.com"        # string (required, with field reference: "http://example.com?id=%{id}" or params, if defined)
      method => "post"                   # string (optional, default = "get")
      headers => {                       # hash (optional)
        "key1" => "value1"
        "key2" => "value2"
      }
      auth => {
        user => "AzureDiamond"
        password => "hunter2"
      }
      params => {                        # hash (optional, available for method => "get" and "post"; if post it will be transformed into body hash and posted as json)
        "key1" => "value1"
        "key2" => "value2"
        "key3" => "%{somefield}"         # sprintf is used implicitly
      }
    }
    json => true                         # boolean (optional, default = true)
    target => "my_key"                   # string (mandatory, no default)
    fallback => {                        # hash describing a default in case of error
      "key1" => "value1"
      "key2" => "value2"
    }
  }
}

Reference : https://github.com/lucashenning/logstash-filter-rest