fluent-plugin-kv-parser 0.1.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +5 -5
  2. data/.travis.yml +2 -3
  3. data/README.md +15 -6
  4. data/fluent-plugin-kv-parser.gemspec +2 -2
  5. data/lib/fluent/plugin/parser_kv.rb +15 -32
  6. data/test/test_kv_parser.rb +40 -46
  7. metadata +12 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: fe95909c734f6a521e7ee45bcc325afe33649f8f
4
- data.tar.gz: a10bd9c7249327b77f91222c60d69444ba0ef0da
2
+ SHA256:
3
+ metadata.gz: f4b7b45b73282ce3360af992bf3f56eab1b6a043ed25d8ee37ffad639079a61f
4
+ data.tar.gz: 63d383d9edc7cf3d46f7d9176e3b7742ca3e26cae50854c27693528d8243a44e
5
5
  SHA512:
6
- metadata.gz: 44d02fcdf9fb09e851daa6f3fa7432c6e528c80ecd3f07ae0eefe27336732340868b34ab69f54418a3a83fc16e20a77d3b3cb35755b2936f23ded15aa3269a87
7
- data.tar.gz: a7d7c9bb816335ebd0ce5d207f9bc662f38a21e676685adeb5047faba07473dc6754e1c2b0778e22a6507f9bba2a34dc0690fab7a99f36bf8145ac9fba850c60
6
+ metadata.gz: 52ae18e5a422586ca55fb78eb3598d9f880a111e22ced3a56e743a55e1a8584ee37d3c374cc8beef631b619f846e0ee23b75f99ba614742c7054d46a9d8565f4
7
+ data.tar.gz: 8b73739ebc3ece980df58d9836e15730c8fd1a0ebdfd52f3174ce7a1d404855d222483684a5b906ae45aa852a94cfc30848f465896087c2e235adaf2b660e765
data/.travis.yml CHANGED
@@ -1,12 +1,11 @@
1
1
  language: ruby
2
2
 
3
3
  rvm:
4
- - 1.9.3
5
- - 2.0.0
6
4
  - 2.1
7
5
  - 2.2
8
6
  - 2.3
9
- - 2.4
7
+ - 2.4.3
8
+ - 2.5.0
10
9
  - ruby-head
11
10
  - rbx-2
12
11
 
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Key-Value Pairs Parser Plugin for [Fluentd](https://github.com/fluent/fluentd)
2
2
 
3
- <img src="https://travis-ci.org/kiyoto/fluent-plugin-kv-parser.svg?branch=master"/>
3
+ [![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-kv-parser.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-kv-parser)
4
4
 
5
5
  ## Overview
6
6
 
@@ -20,6 +20,13 @@ It is parsed as
20
20
 
21
21
  with the event's time being `2013-01-01T12:34:00`
22
22
 
23
+ ## Requirements
24
+
25
+ | fluent-plugin-kv-parser | fluentd | ruby |
26
+ |-------------------------|------------|--------|
27
+ | >= 1.0.0 | >= v0.14.0 | >= 2.1 |
28
+ | < 1.0.0 | >= v0.12.0 | >= 1.9 |
29
+
23
30
  ## How to Install and Use
24
31
 
25
32
  For Fluentd,
@@ -40,15 +47,17 @@ For example, using `in_tcp` with the following configuration:
40
47
 
41
48
  ```aconf
42
49
  <source>
43
- type tcp
50
+ @type tcp
44
51
  port 24225
45
52
  tag kv_log
46
- format kv
47
- time_key my_time
48
- types k1:integer,my_time:time
53
+ <parse>
54
+ @type kv
55
+ time_key my_time
56
+ types k1:integer,my_time:time
57
+ </parse>
49
58
  </source>
50
59
  <match kv_log>
51
- type stdout
60
+ @type stdout
52
61
  </match>
53
62
  ```
54
63
 
data/fluent-plugin-kv-parser.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-kv-parser"
7
- spec.version = "0.1.0"
7
+ spec.version = "1.0.0"
8
8
  spec.description = 'Fluentd parser plugin to parse key value pairs'
9
9
  spec.authors = ["kiyoto"]
10
10
  spec.email = ["kiyoto@treasure-data.com"]
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
19
19
 
20
20
  spec.add_development_dependency "rake"
21
21
  spec.add_development_dependency "test-unit", "> 3.0"
22
- spec.add_runtime_dependency "fluentd", '~> 0.12.0'
22
+ spec.add_runtime_dependency "fluentd", '> 0.14.0', '< 2'
23
23
  end
data/lib/fluent/plugin/parser_kv.rb CHANGED
@@ -1,14 +1,16 @@
1
+ require "fluent/plugin/parser"
2
+
1
3
  module Fluent
2
- class TextParser
3
- class KVParser < Parser
4
- include Configurable
5
- include TypeConverter
4
+ module Plugin
5
+ class KVParser < Fluent::Plugin::Parser
6
+ Fluent::Plugin.register_parser("kv", self)
7
+
8
+ config_param :kv_delimiter, :string, default: '/\s+/'
9
+ config_param :kv_char, :string, default: '='
6
10
 
7
- config_param :kv_delimiter, :string, :default => '/[\t\s]+/'
8
- config_param :kv_char, :string, :default => '='
9
- config_param :time_key, :string, :default => 'time'
11
+ config_set_default :time_key, "time"
10
12
 
11
- def configure(conf={})
13
+ def configure(conf)
12
14
  super
13
15
  if @kv_delimiter[0] == '/' and @kv_delimiter[-1] == '/'
14
16
  @kv_delimiter = Regexp.new(@kv_delimiter[1..-2])
@@ -18,34 +20,15 @@ module Fluent
18
20
  def parse(text)
19
21
  record = {}
20
22
  text.split(@kv_delimiter).each do |kv|
21
- k, v = kv.split(@kv_char, 2)
22
- record[k] = v
23
+ key, value = kv.split(@kv_char, 2)
24
+ record[key] = value
23
25
  end
24
26
 
25
- convert_field_type!(record) if @type_converters
26
- time = record.delete(@time_key)
27
- if time.nil?
28
- time = Engine.now
29
- elsif time.respond_to?(:to_i)
30
- time = time.to_i
31
- else
32
- raise RuntimeError, "The #{@time_key}=#{time} is a bad time field"
33
- end
27
+ time = parse_time(record)
28
+ time, record = convert_values(time, record)
34
29
 
35
30
  yield time, record
36
31
  end
37
-
38
- private
39
-
40
- def convert_field_type!(record)
41
- @type_converters.each_key { |key|
42
- if value = record[key]
43
- record[key] = convert_type(key, value)
44
- end
45
- }
46
- end
47
-
48
32
  end
49
- register_template('kv', Proc.new { KVParser.new })
50
33
  end
51
- end
34
+ end
data/test/test_kv_parser.rb CHANGED
@@ -1,73 +1,67 @@
1
1
  require 'fluent/test'
2
- require 'fluent/parser'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/test/helpers'
3
4
  require 'fluent/plugin/parser_kv'
4
5
 
5
6
  class KVParserTest < ::Test::Unit::TestCase
6
-
7
- def create_driver(conf = {})
8
- Fluent::Test::ParserTestDriver.new(Fluent::TextParser::KVParser).configure(conf)
9
- end
7
+ include Fluent::Test::Helpers
10
8
 
11
9
  def setup
12
- @timezone = ENV["TZ"]
13
- ENV["TZ"] = "UTC"
10
+ Fluent::Test.setup
14
11
  end
15
12
 
16
- def teardown
17
- ENV["TZ"] = @timezone
13
+ def create_driver(conf={})
14
+ Fluent::Test::Driver::Parser.new(Fluent::Plugin::KVParser).configure(conf)
18
15
  end
19
16
 
20
- def test_basic
17
+ data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
18
+ "multiple space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
19
+ "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
20
+ "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
21
+ "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
22
+ test "parse" do |(text, expected)|
21
23
  d = create_driver
22
- parser = d.instance
23
- parser.parse("k1=v1 k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
24
- parser.parse("k1=v1 k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
25
- parser.parse("k2=v2 k1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
26
- parser.parse("k2=v2\tk1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
27
- parser.parse("k2=v2\t k1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
24
+ d.instance.parse(text) do |_time, record|
25
+ assert_equal(expected, record)
26
+ end
28
27
  end
29
28
 
30
- def test_with_types
29
+ test "parse with types" do
31
30
  d = create_driver("types" => "k1:integer")
32
- parser = d.instance
33
- parser.parse("k1=100") {|_, v| assert_equal(100, v["k1"])}
31
+ d.instance.parse("k1=100") do |_time, record|
32
+ assert_equal({ "k1" => 100 }, record)
33
+ end
34
34
  end
35
35
 
36
- def test_with_time
36
+ test "parse with time" do
37
37
  d = create_driver("types" => "time:time")
38
- parser = d.instance
39
- parser.parse("k1=foo time=1970-01-01T01:00:00") {|time, v|
40
- assert_equal(3600, time)
41
- assert_equal("foo", v["k1"])
42
- }
38
+ d.instance.parse("k1=foo time=1970-01-01T01:00:00") do |time, record|
39
+ assert_equal(event_time("1970-01-01T01:00:0"), time)
40
+ assert_equal({ "k1" => "foo" }, record)
41
+ end
43
42
  end
44
43
 
45
- def test_with_custom_time_key
44
+ test "parse with custom time_key" do
46
45
  d = create_driver("time_key" => "my_time", "types" => "my_time:time")
47
- parser = d.instance
48
- parser.parse("k1=foo my_time=1970-01-01T01:00:00") {|time, v|
49
- assert_equal(3600, time)
50
- assert_equal("foo", v["k1"])
51
- }
52
- end
53
-
54
- def test_custom_delimiter
55
- d = create_driver("kv_delimiter" => "|")
56
- parser = d.instance
57
- parser.parse("k1=v1|k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
46
+ d.instance.parse("k1=foo my_time=1970-01-01T01:00:00") do |time, record|
47
+ assert_equal(event_time("1970-01-01T01:00:0"), time)
48
+ assert_equal({ "k1" => "foo" }, record)
49
+ end
58
50
  end
59
51
 
60
- def test_custom_delimiter2
61
- d = create_driver("kv_delimiter" => "/[@ ]/")
62
- parser = d.instance
63
- parser.parse("k1=v1@k2=v2 k3=v3") {|_, v|
64
- assert_equal({"k1"=>"v1", "k2"=>"v2", "k3"=>"v3"}, v)
65
- }
52
+ data("pipe" => ["|", "k1=v1|k2=v2", {"k1" => "v1", "k2" => "v2" }],
53
+ "regexp" => ["/[@ ]/", "k1=v1@k2=v2 k3=v3", { "k1" => "v1", "k2" => "v2", "k3" => "v3" }])
54
+ test "parse with custom kv_delimiter" do |(delimiter, text, expected)|
55
+ d = create_driver("kv_delimiter" => delimiter)
56
+ d.instance.parse(text) do |_time, record|
57
+ assert_equal(expected, record)
58
+ end
66
59
  end
67
60
 
68
- def test_custom_kv_char
61
+ test "parse with custom kv_char" do
69
62
  d = create_driver("kv_char" => "#")
70
- parser = d.instance
71
- parser.parse("k1#v1 k2#v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
63
+ d.instance.parse("k1#v1 k2#v2") do |_time, record|
64
+ assert_equal({ "k1" => "v1", "k2" => "v2" }, record)
65
+ end
72
66
  end
73
67
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kv-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - kiyoto
@@ -42,16 +42,22 @@ dependencies:
42
42
  name: fluentd
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.14.0
48
+ - - "<"
46
49
  - !ruby/object:Gem::Version
47
- version: 0.12.0
50
+ version: '2'
48
51
  type: :runtime
49
52
  prerelease: false
50
53
  version_requirements: !ruby/object:Gem::Requirement
51
54
  requirements:
52
- - - "~>"
55
+ - - ">"
56
+ - !ruby/object:Gem::Version
57
+ version: 0.14.0
58
+ - - "<"
53
59
  - !ruby/object:Gem::Version
54
- version: 0.12.0
60
+ version: '2'
55
61
  description: Fluentd parser plugin to parse key value pairs
56
62
  email:
57
63
  - kiyoto@treasure-data.com
@@ -86,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
86
92
  version: '0'
87
93
  requirements: []
88
94
  rubyforge_project:
89
- rubygems_version: 2.6.14
95
+ rubygems_version: 2.7.3
90
96
  signing_key:
91
97
  specification_version: 4
92
98
  summary: Fluentd parser plugin to parse key value pairs