diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..12ed4ff1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,6 @@ +FROM puppet/pdk:latest + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..a7193616 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,38 @@ +# devcontainer + + +For format details, see https://aka.ms/devcontainer.json. + +For config options, see the README at: +https://github.com/microsoft/vscode-dev-containers/tree/v0.140.1/containers/puppet + +``` json +{ + "name": "Puppet Development Kit (Community)", + "dockerFile": "Dockerfile", + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "bash", + } + } + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "puppet.puppet-vscode", + "rebornix.Ruby" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "pdk --version", +} +``` + + + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..fe7a8b12 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "Puppet Development Kit (Community)", + "dockerFile": "Dockerfile", + + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "bash", + } + } + }, + + "extensions": [ + "puppet.puppet-vscode", + "rebornix.Ruby" + ] +} diff --git a/.fixtures.yml b/.fixtures.yml index d9c4a044..75499b6f 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -1,9 +1,10 @@ fixtures: repositories: + yumrepo_core: + repo: "https://github.com/puppetlabs/puppetlabs-yumrepo_core.git" + puppet_version: ">= 6.0.0" stdlib: "https://github.com/puppetlabs/puppetlabs-stdlib.git" - apt: - repo: "https://github.com/puppetlabs/puppetlabs-apt.git" - ref: '4.2.0' + apt: "https://github.com/puppetlabs/puppetlabs-apt.git" powershell: "https://github.com/puppetlabs/puppetlabs-powershell.git" archive: "https://github.com/voxpupuli/puppet-archive.git" symlinks: diff --git a/.gitattributes b/.gitattributes index 543dd6ad..9032a014 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,4 @@ *.erb eol=lf *.pp eol=lf *.sh eol=lf +*.epp eol=lf diff --git a/.github/workflows/pdk-test-unit.yml b/.github/workflows/pdk-test-unit.yml new file mode 100644 index 00000000..562afd57 --- /dev/null +++ b/.github/workflows/pdk-test-unit.yml @@ -0,0 +1,22 @@ +name: Run pdk test unit + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v3 + + - name: Run unit tests + uses: puppets-epic-show-theatre/action-pdk-test-unit@v1 + with: + puppet-version: "" + # [optional] + # A string indicating the Puppet version to validate against, such as "5.4.2" or "5.5". + pe-version: "" + # [optional] + # A string indicating the PE version to validate against, such as "2017.3.5" or "2018.1". diff --git a/.github/workflows/pdk-validate.yml b/.github/workflows/pdk-validate.yml new file mode 100644 index 00000000..cd5f495b --- /dev/null +++ b/.github/workflows/pdk-validate.yml @@ -0,0 +1,22 @@ +name: Run pdk static validation + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v3 + + - name: Run pdk validate + uses: puppets-epic-show-theatre/action-pdk-validate@v1 + with: + puppet-version: "" + # [optional] + # A string indicating the Puppet version to validate against, such as "5.4.2" or "5.5". + pe-version: "" + # [optional] + # A string indicating the PE version to validate against, such as "2017.3.5" or "2018.1". diff --git a/.gitignore b/.gitignore index 78d9ade0..988dcbbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.git/ .*.sw[op] .metadata .yardoc @@ -13,9 +14,15 @@ /Gemfile.lock /junit/ /log/ -/log/ /pkg/ /spec/fixtures/manifests/ /spec/fixtures/modules/ /tmp/ /vendor/ +/convert_report.txt +/update_report.txt +.DS_Store +.project +.envrc +/inventory.yaml +/spec/fixtures/litmus_inventory.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..6d5e786c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,53 @@ +--- +stages: + - syntax + - unit + +default: + cache: + paths: + - vendor/bundle + + before_script: &before_script + - bundle -v + - rm Gemfile.lock || true + - "# Update system gems if requested. This is useful to temporarily workaround troubles in the test runner" + - "# Set `rubygems_version` in the .sync.yml to set a value" + - "# Ignore exit code of SIGPIPE'd yes to not fail with shell's pipefail set" + - '[ -z "$RUBYGEMS_VERSION" ] || (yes || true) | gem update --system $RUBYGEMS_VERSION' + - gem --version + - bundle -v + - bundle install --without system_tests --path vendor/bundle --jobs $(nproc) + +validate lint check rubocop-Ruby 2.5.7-Puppet ~> 6: + stage: syntax + image: ruby:2.5.7 + script: + - bundle exec rake validate lint check rubocop + variables: + PUPPET_GEM_VERSION: '~> 6' + +parallel_spec-Ruby 2.5.7-Puppet ~> 6: + stage: unit + image: ruby:2.5.7 + script: + - bundle exec rake parallel_spec + variables: + PUPPET_GEM_VERSION: '~> 6' + +validate lint check rubocop-Ruby 2.7.2-Puppet ~> 7: + stage: syntax + image: ruby:2.7.2 + script: + - bundle exec rake validate lint check rubocop + variables: + PUPPET_GEM_VERSION: '~> 7' + +parallel_spec-Ruby 2.7.2-Puppet ~> 7: + stage: unit + image: ruby:2.7.2 + script: + - bundle exec rake parallel_spec + variables: + PUPPET_GEM_VERSION: '~> 7' + diff --git a/.pdkignore b/.pdkignore new file mode 100644 index 00000000..c538bea8 --- /dev/null +++ b/.pdkignore @@ -0,0 +1,47 @@ +.git/ +.*.sw[op] +.metadata +.yardoc +.yardwarns +*.iml +/.bundle/ +/.idea/ +/.vagrant/ +/coverage/ +/bin/ +/doc/ +/Gemfile.local +/Gemfile.lock +/junit/ +/log/ +/pkg/ +/spec/fixtures/manifests/ +/spec/fixtures/modules/ +/tmp/ +/vendor/ +/convert_report.txt +/update_report.txt +.DS_Store +.project +.envrc +/inventory.yaml +/spec/fixtures/litmus_inventory.yaml +/appveyor.yml +/.editorconfig +/.fixtures.yml +/Gemfile +/.gitattributes +/.gitignore +/.gitlab-ci.yml +/.pdkignore +/.puppet-lint.rc +/Rakefile +/rakelib/ +/.rspec +/.rubocop.yml +/.travis.yml +/.yardopts +/spec/ +/.vscode/ +/.sync.yml +/.devcontainer/ diff --git a/.puppet-lint.rc b/.puppet-lint.rc new file mode 100644 index 00000000..cc96ece0 --- /dev/null +++ b/.puppet-lint.rc @@ -0,0 +1 @@ +--relative diff --git a/.rubocop.yml b/.rubocop.yml index 72d2f75e..31e8248f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,18 +1,24 @@ --- -require: rubocop-rspec +require: +- rubocop-performance +- rubocop-rspec AllCops: - TargetRubyVersion: '2.1' + DisplayCopNames: true + TargetRubyVersion: '2.5' Include: - - "./**/*.rb" + - "**/*.rb" Exclude: - bin/* - ".vendor/**/*" - - Gemfile - - Rakefile + - "**/Gemfile" + - "**/Rakefile" - pkg/**/* - spec/fixtures/**/* - vendor/**/* -Metrics/LineLength: + - "**/Puppetfile" + - "**/Vagrantfile" + - "**/Guardfile" +Layout/LineLength: Description: People have wide screens, use them. Max: 200 RSpec/BeforeAfterAll: @@ -23,6 +29,9 @@ RSpec/BeforeAfterAll: RSpec/HookArgument: Description: Prefer explicit :each argument, matching existing module's style EnforcedStyle: each +RSpec/DescribeSymbol: + Exclude: + - spec/unit/facter/**/*.rb Style/BlockDelimiters: Description: Prefer braces for chaining. Mostly an aesthetical choice. Better to be consistent then. @@ -55,25 +64,185 @@ Style/TrailingCommaInArguments: Description: Prefer always trailing comma on multiline argument lists. This makes diffs, and re-ordering nicer. EnforcedStyleForMultiline: comma -Style/TrailingCommaInLiteral: +Style/TrailingCommaInArrayLiteral: Description: Prefer always trailing comma on multiline literals. This makes diffs, and re-ordering nicer. EnforcedStyleForMultiline: comma Style/SymbolArray: Description: Using percent style obscures symbolic intent of array's contents. EnforcedStyle: brackets -Style/EndOfLine: - Enabled: false +RSpec/MessageSpies: + EnforcedStyle: receive +Style/Documentation: + Exclude: + - lib/puppet/parser/functions/**/* + - spec/**/* +Style/WordArray: + EnforcedStyle: brackets +Performance/AncestorsInclude: + Enabled: true +Performance/BigDecimalWithNumericArgument: + Enabled: true +Performance/BlockGivenWithExplicitBlock: + Enabled: true +Performance/CaseWhenSplat: + Enabled: true +Performance/ConstantRegexp: + Enabled: true +Performance/MethodObjectAsBlock: + Enabled: true +Performance/RedundantSortBlock: + Enabled: true +Performance/RedundantStringChars: + Enabled: true +Performance/ReverseFirst: + Enabled: true +Performance/SortReverse: + Enabled: true +Performance/Squeeze: + Enabled: true +Performance/StringInclude: + Enabled: true +Performance/Sum: + Enabled: true Style/CollectionMethods: Enabled: true Style/MethodCalledOnDoEndBlock: Enabled: true Style/StringMethods: Enabled: true +Bundler/InsecureProtocolSource: + Enabled: false +Gemspec/DuplicatedAssignment: + Enabled: false +Gemspec/OrderedDependencies: + Enabled: false +Gemspec/RequiredRubyVersion: + Enabled: false +Gemspec/RubyVersionGlobalsUsage: + Enabled: false +Layout/ArgumentAlignment: + Enabled: false +Layout/BeginEndAlignment: + Enabled: false +Layout/ClosingHeredocIndentation: + Enabled: false +Layout/EmptyComment: + Enabled: false +Layout/EmptyLineAfterGuardClause: + Enabled: false +Layout/EmptyLinesAroundArguments: + Enabled: false +Layout/EmptyLinesAroundAttributeAccessor: + Enabled: false +Layout/EndOfLine: + Enabled: false +Layout/FirstArgumentIndentation: + Enabled: false +Layout/HashAlignment: + Enabled: false +Layout/HeredocIndentation: + Enabled: false +Layout/LeadingEmptyLines: + Enabled: false +Layout/SpaceAroundMethodCallOperator: + Enabled: false +Layout/SpaceInsideArrayLiteralBrackets: + Enabled: false +Layout/SpaceInsideReferenceBrackets: + Enabled: false +Lint/BigDecimalNew: + Enabled: false +Lint/BooleanSymbol: + Enabled: false +Lint/ConstantDefinitionInBlock: + Enabled: false +Lint/DeprecatedOpenSSLConstant: + Enabled: false +Lint/DisjunctiveAssignmentInConstructor: + Enabled: false +Lint/DuplicateElsifCondition: + Enabled: false +Lint/DuplicateRequire: + Enabled: false +Lint/DuplicateRescueException: + Enabled: false +Lint/EmptyConditionalBody: + Enabled: false +Lint/EmptyFile: + Enabled: false +Lint/ErbNewArguments: + Enabled: false +Lint/FloatComparison: + Enabled: false +Lint/HashCompareByIdentity: + Enabled: false +Lint/IdentityComparison: + Enabled: false +Lint/InterpolationCheck: + Enabled: false +Lint/MissingCopEnableDirective: + Enabled: false +Lint/MixedRegexpCaptureTypes: + Enabled: false +Lint/NestedPercentLiteral: + Enabled: false +Lint/NonDeterministicRequireOrder: + Enabled: false +Lint/OrderedMagicComments: + Enabled: false +Lint/OutOfRangeRegexpRef: + Enabled: false +Lint/RaiseException: + Enabled: false +Lint/RedundantCopEnableDirective: + Enabled: false +Lint/RedundantRequireStatement: + Enabled: false +Lint/RedundantSafeNavigation: + Enabled: false +Lint/RedundantWithIndex: + Enabled: false +Lint/RedundantWithObject: + Enabled: false +Lint/RegexpAsCondition: + Enabled: false +Lint/ReturnInVoidContext: + Enabled: false +Lint/SafeNavigationConsistency: + Enabled: false +Lint/SafeNavigationWithEmpty: + Enabled: false +Lint/SelfAssignment: + Enabled: false +Lint/SendWithMixinArgument: + Enabled: false +Lint/ShadowedArgument: + Enabled: false +Lint/StructNewOverride: + Enabled: false +Lint/ToJSON: + Enabled: false +Lint/TopLevelReturnWithArgument: + Enabled: false +Lint/TrailingCommaInAttributeDeclaration: + Enabled: false +Lint/UnreachableLoop: + Enabled: false +Lint/UriEscapeUnescape: + Enabled: false +Lint/UriRegexp: + Enabled: false +Lint/UselessMethodDefinition: + Enabled: false +Lint/UselessTimes: + Enabled: false Metrics/AbcSize: Enabled: false Metrics/BlockLength: Enabled: false +Metrics/BlockNesting: + Enabled: false Metrics/ClassLength: Enabled: false Metrics/CyclomaticComplexity: @@ -86,19 +255,265 @@ Metrics/ParameterLists: Enabled: false Metrics/PerceivedComplexity: Enabled: false +Migration/DepartmentName: + Enabled: false +Naming/AccessorMethodName: + Enabled: false +Naming/BlockParameterName: + Enabled: false +Naming/HeredocDelimiterCase: + Enabled: false +Naming/HeredocDelimiterNaming: + Enabled: false +Naming/MemoizedInstanceVariableName: + Enabled: false +Naming/MethodParameterName: + Enabled: false +Naming/RescuedExceptionsVariableName: + Enabled: false +Naming/VariableNumber: + Enabled: false +Performance/BindCall: + Enabled: false +Performance/DeletePrefix: + Enabled: false +Performance/DeleteSuffix: + Enabled: false +Performance/InefficientHashSearch: + Enabled: false +Performance/UnfreezeString: + Enabled: false +Performance/UriDefaultParser: + Enabled: false +RSpec/Be: + Enabled: false +RSpec/Capybara/CurrentPathExpectation: + Enabled: false +RSpec/Capybara/FeatureMethods: + Enabled: false +RSpec/Capybara/VisibilityMatcher: + Enabled: false +RSpec/ContextMethod: + Enabled: false +RSpec/ContextWording: + Enabled: false RSpec/DescribeClass: Enabled: false +RSpec/EmptyHook: + Enabled: false +RSpec/EmptyLineAfterExample: + Enabled: false +RSpec/EmptyLineAfterExampleGroup: + Enabled: false +RSpec/EmptyLineAfterHook: + Enabled: false RSpec/ExampleLength: Enabled: false -RSpec/MessageExpectation: +RSpec/ExampleWithoutDescription: + Enabled: false +RSpec/ExpectChange: + Enabled: false +RSpec/ExpectInHook: + Enabled: false +RSpec/FactoryBot/AttributeDefinedStatically: + Enabled: false +RSpec/FactoryBot/CreateList: + Enabled: false +RSpec/FactoryBot/FactoryClassName: + Enabled: false +RSpec/HooksBeforeExamples: + Enabled: false +RSpec/ImplicitBlockExpectation: + Enabled: false +RSpec/ImplicitSubject: + Enabled: false +RSpec/LeakyConstantDeclaration: + Enabled: false +RSpec/LetBeforeExamples: + Enabled: false +RSpec/MissingExampleGroupArgument: Enabled: false RSpec/MultipleExpectations: Enabled: false +RSpec/MultipleMemoizedHelpers: + Enabled: false +RSpec/MultipleSubjects: + Enabled: false RSpec/NestedGroups: Enabled: false +RSpec/PredicateMatcher: + Enabled: false +RSpec/ReceiveCounts: + Enabled: false +RSpec/ReceiveNever: + Enabled: false +RSpec/RepeatedExampleGroupBody: + Enabled: false +RSpec/RepeatedExampleGroupDescription: + Enabled: false +RSpec/RepeatedIncludeExample: + Enabled: false +RSpec/ReturnFromStub: + Enabled: false +RSpec/SharedExamples: + Enabled: false +RSpec/StubbedMock: + Enabled: false +RSpec/UnspecifiedException: + Enabled: false +RSpec/VariableDefinition: + Enabled: false +RSpec/VoidExpect: + Enabled: false +RSpec/Yield: + Enabled: false +Security/Open: + Enabled: false +Style/AccessModifierDeclarations: + Enabled: false +Style/AccessorGrouping: + Enabled: false Style/AsciiComments: Enabled: false +Style/BisectedAttrAccessor: + Enabled: false +Style/CaseLikeIf: + Enabled: false +Style/ClassEqualityComparison: + Enabled: false +Style/ColonMethodDefinition: + Enabled: false +Style/CombinableLoops: + Enabled: false +Style/CommentedKeyword: + Enabled: false +Style/Dir: + Enabled: false +Style/DoubleCopDisableDirective: + Enabled: false +Style/EmptyBlockParameter: + Enabled: false +Style/EmptyLambdaParameter: + Enabled: false +Style/Encoding: + Enabled: false +Style/EvalWithLocation: + Enabled: false +Style/ExpandPathArguments: + Enabled: false +Style/ExplicitBlockArgument: + Enabled: false +Style/ExponentialNotation: + Enabled: false +Style/FloatDivision: + Enabled: false +Style/FrozenStringLiteralComment: + Enabled: false +Style/GlobalStdStream: + Enabled: false +Style/HashAsLastArrayItem: + Enabled: false +Style/HashLikeCase: + Enabled: false +Style/HashTransformKeys: + Enabled: false +Style/HashTransformValues: + Enabled: false Style/IfUnlessModifier: Enabled: false +Style/KeywordParametersOrder: + Enabled: false +Style/MinMax: + Enabled: false +Style/MixinUsage: + Enabled: false +Style/MultilineWhenThen: + Enabled: false +Style/NegatedUnless: + Enabled: false +Style/NumericPredicate: + Enabled: false +Style/OptionalBooleanParameter: + Enabled: false +Style/OrAssignment: + Enabled: false +Style/RandomWithOffset: + Enabled: false +Style/RedundantAssignment: + Enabled: false +Style/RedundantCondition: + Enabled: false +Style/RedundantConditional: + Enabled: false +Style/RedundantFetchBlock: + Enabled: false +Style/RedundantFileExtensionInRequire: + Enabled: false +Style/RedundantRegexpCharacterClass: + Enabled: false +Style/RedundantRegexpEscape: + Enabled: false +Style/RedundantSelfAssignment: + Enabled: false +Style/RedundantSort: + Enabled: false +Style/RescueStandardError: + Enabled: false +Style/SingleArgumentDig: + Enabled: false +Style/SlicingWithRange: + Enabled: false +Style/SoleNestedConditional: + Enabled: false +Style/StderrPuts: + Enabled: false +Style/StringConcatenation: + Enabled: false +Style/Strip: + Enabled: false Style/SymbolProc: Enabled: false +Style/TrailingBodyOnClass: + Enabled: false +Style/TrailingBodyOnMethodDefinition: + Enabled: false +Style/TrailingBodyOnModule: + Enabled: false +Style/TrailingCommaInHashLiteral: + Enabled: false +Style/TrailingMethodEndStatement: + Enabled: false +Style/UnpackFirst: + Enabled: false +Lint/DuplicateBranch: + Enabled: false +Lint/DuplicateRegexpCharacterClassElement: + Enabled: false +Lint/EmptyBlock: + Enabled: false +Lint/EmptyClass: + Enabled: false +Lint/NoReturnInBeginEndBlocks: + Enabled: false +Lint/ToEnumArguments: + Enabled: false +Lint/UnexpectedBlockArity: + Enabled: false +Lint/UnmodifiedReduceAccumulator: + Enabled: false +Performance/CollectionLiteralInLoop: + Enabled: false +Style/ArgumentsForwarding: + Enabled: false +Style/CollectionCompact: + Enabled: false +Style/DocumentDynamicEvalDefinition: + Enabled: false +Style/NegatedIfElseCondition: + Enabled: false +Style/NilLambda: + Enabled: false +Style/RedundantArgument: + Enabled: false +Style/SwapValues: + Enabled: false diff --git a/.sync.yml b/.sync.yml new file mode 100644 index 00000000..532f777c --- /dev/null +++ b/.sync.yml @@ -0,0 +1,12 @@ +mock_with: ':mocha' + +.travis.yml: + user: pcfens + secure: "z1SbP/Hisr5k66XL/ACLsZ/fG7cCpwl8apjZzt/YciWizwReioU2EkLr5tvXdUC10aIH6H7XBdA9XwPqwXa81cIqcdIHlRMIbosMUGYaXcUm1xhctB3GvEDqsxFqdZSHYXax+IR6Wt507Eop+iU3S5pf/zJcp4uSKQVapCMoeVCEQYLRwllgeaqtEUZwqOUwPk31C4YZxwrzmgbIVyXmPrp3SDToXaQm4S4RkayOqHH2lYi8isz3IPPQvDZY5681TBpo35AbsIRbhiLzGlBHbgRaE2dz7J1Gs8MBGFyrtDaPtc9UpbgEmyxgmaPs3NIeZkmfVoosjt2AHRsoMZB7ntaPAQ20mk44ugMhxd5HX8t7QdLPiYQqgA3O4QfKraxPzdEjYVs9Pf7BBgY4JpGSOAD3dlWNK0U40MzKe74cj6dshg9SfIdyf3M3MmI0KIIvdKhpgl8mSIL8MCWjnYYNpQMQDFgyrXvePnkPVlt7zlBxn+LJFFx3VLGNfSWbKavITM/nrvjpFkQZ34mPHTtTUYnT6HVehtwPd5x6ILqYcppEeeiloa4uLWhW/vg0wAOdOBv2IALdAqRMC56ODPK33gFRkX+CclsegtOh2In407njbXXZBQrY5h3SXuEVxZcFhGVTxJIV29viuWFSm7VF0a7IUmEbVrM23bqeaM+aOgs=" + branches: + only: + - master + - /^v\d/ + +common: + disable_legacy_facts: true diff --git a/.travis.yml b/.travis.yml old mode 100644 new mode 100755 index 1271c07e..195ec2c4 --- a/.travis.yml +++ b/.travis.yml @@ -1,48 +1,54 @@ --- +os: linux +dist: xenial language: ruby cache: bundler before_install: - bundle -v - - rm Gemfile.lock || true - - gem update --system - - gem update bundler + - rm -f Gemfile.lock + - "# Update system gems if requested. This is useful to temporarily workaround troubles in the test runner" + - "# See https://github.com/puppetlabs/pdk-templates/commit/705154d5c437796b821691b707156e1b056d244f for an example of how this was used" + - "# Ignore exit code of SIGPIPE'd yes to not fail with shell's pipefail set" + - '[ -z "$RUBYGEMS_VERSION" ] || (yes || true) | gem update --system $RUBYGEMS_VERSION' - gem --version - bundle -v script: - - "bundle exec rake validate metadata_lint lint spec SPEC_OPTS='--format documentation'" -matrix: + - 'bundle exec rake $CHECK' +bundler_args: --without system_tests +rvm: + - 2.5.7 +stages: + - static + - spec + - acceptance + - + if: tag =~ ^v\d + name: deploy +jobs: fast_finish: true include: - - rvm: 2.3.1 - dist: trusty - env: PUPPET_INSTALL_TYPE=agent BEAKER_debug=true - script: bundle exec rake beaker - services: docker - sudo: required - - rvm: 2.3.1 - dist: trusty - env: BEAKER_set=centos-7 PUPPET_INSTALL_TYPE=agent BEAKER_debug=true - script: bundle exec rake beaker - services: docker - sudo: required - - rvm: 2.3.1 - bundler_args: --without system_tests - env: STRICT_VARIABLES="yes" - script: bundle exec rake rubocop - - rvm: 2.1.9 - bundler_args: --without system_tests - env: PUPPET_GEM_VERSION="~> 4.0" STRICT_VARIABLES=yes - - rvm: 2.4.0 - bundler_args: --without system_tests - env: PUPPET_GEM_VERSION="~> 5.0" STRICT_VARIABLES=yes DEPLOY_TO_FORGE=yes + - + env: CHECK="validate lint check rubocop" + stage: static + - + env: PUPPET_GEM_VERSION="~> 6.0" CHECK=parallel_spec + rvm: 2.5.7 + stage: spec + - + env: DEPLOY_TO_FORGE=yes + stage: deploy +branches: + only: + - main + - /^v\d/ +notifications: + email: false deploy: provider: puppetforge - user: "pcfens" + username: pcfens password: secure: "z1SbP/Hisr5k66XL/ACLsZ/fG7cCpwl8apjZzt/YciWizwReioU2EkLr5tvXdUC10aIH6H7XBdA9XwPqwXa81cIqcdIHlRMIbosMUGYaXcUm1xhctB3GvEDqsxFqdZSHYXax+IR6Wt507Eop+iU3S5pf/zJcp4uSKQVapCMoeVCEQYLRwllgeaqtEUZwqOUwPk31C4YZxwrzmgbIVyXmPrp3SDToXaQm4S4RkayOqHH2lYi8isz3IPPQvDZY5681TBpo35AbsIRbhiLzGlBHbgRaE2dz7J1Gs8MBGFyrtDaPtc9UpbgEmyxgmaPs3NIeZkmfVoosjt2AHRsoMZB7ntaPAQ20mk44ugMhxd5HX8t7QdLPiYQqgA3O4QfKraxPzdEjYVs9Pf7BBgY4JpGSOAD3dlWNK0U40MzKe74cj6dshg9SfIdyf3M3MmI0KIIvdKhpgl8mSIL8MCWjnYYNpQMQDFgyrXvePnkPVlt7zlBxn+LJFFx3VLGNfSWbKavITM/nrvjpFkQZ34mPHTtTUYnT6HVehtwPd5x6ILqYcppEeeiloa4uLWhW/vg0wAOdOBv2IALdAqRMC56ODPK33gFRkX+CclsegtOh2In407njbXXZBQrY5h3SXuEVxZcFhGVTxJIV29viuWFSm7VF0a7IUmEbVrM23bqeaM+aOgs=" on: tags: true all_branches: true condition: "$DEPLOY_TO_FORGE = yes" -notifications: - email: false diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..2f1e4f73 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "puppet.puppet-vscode", + "rebornix.Ruby" + ] +} diff --git a/.yardopts b/.yardopts index 3687f518..29c933bc 100644 --- a/.yardopts +++ b/.yardopts @@ -1,2 +1 @@ --markup markdown ---output-dir docs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c8ad33a..148123fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,249 @@ Changelog ========= ## Unreleased -[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.4.0...HEAD) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.13.0...HEAD) + +## [v4.14.0](https://github.com/pcfens/puppet-filebeat/tree/v4.14.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.13.0...v4.14.0) + +- Template SSL Correction [\#322](https://github.com/pcfens/puppet-filebeat/pull/322) +- Avoid deprecated has_ssl method [\#324](https://github.com/pcfens/puppet-filebeat/pull/324) +- Add missing options to JSON decoding config [\#326](https://github.com/pcfens/puppet-filebeat/pull/326) + +## [v4.13.0](https://github.com/pcfens/puppet-filebeat/tree/v4.13.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.12.0...v4.13.0) + +- Support the filebeat version 8 validation[\#318](https://github.com/pcfens/puppet-filebeat/pull/318) +- Support queue configuration [\#313](https://github.com/pcfens/puppet-filebeat/pull/313) +- Support Windows 2016 [\#316](https://github.com/pcfens/puppet-filebeat/pull/316) +- Support queue configuration [\#313](https://github.com/pcfens/puppet-filebeat/pull/313) +- Support file exclusion with filestream [\#311](https://github.com/pcfens/puppet-filebeat/pull/311) +- Replace legacy facts with structured facts [\#319](https://github.com/pcfens/puppet-filebeat/pull/319) +- Fix tests [\#319](https://github.com/pcfens/puppet-filebeat/pull/319) +- Update PDK [\#319](https://github.com/pcfens/puppet-filebeat/pull/319) + +## [v4.12.0](https://github.com/pcfens/puppet-filebeat/tree/v4.12.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.11.0...v4.12.0) + +- Various lint and bugfixes +- Add filestream support [\#303](https://github.com/pcfens/puppet-filebeat/pull/303) +- Add journald support [\#302](https://github.com/pcfens/puppet-filebeat/pull/302) +- Add syslog support [\#301](https://github.com/pcfens/puppet-filebeat/pull/301) +- Allow Filebeat 8 to be installed [\#300](https://github.com/pcfens/puppet-filebeat/pull/300) +- Add `filebeat.overwrite_pipelines` option [\#286](https://github.com/pcfens/puppet-filebeat/pull/286) +- Fix autodiscover apply location [\#291](https://github.com/pcfens/puppet-filebeat/pull/291) +- Add SSL support for inputs [\#295](https://github.com/pcfens/puppet-filebeat/pull/295) +- Add support for Solaris [\#296](https://github.com/pcfens/puppet-filebeat/pull/296) +- Add support for filestreams [\#297](https://github.com/pcfens/puppet-filebeat/pull/297) + +## [v4.11.0](https://github.com/pcfens/puppet-filebeat/tree/v4.11.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.10.0...v4.11.0) + +- Support setting registry options [\#284](https://github.com/pcfens/puppet-filebeat/pull/284) + +## [v4.10.0](https://github.com/pcfens/puppet-filebeat/tree/v4.10.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.9.0...v4.10.0) + +- Add a parameter to skip managing the filebeat package [\#281](https://github.com/pcfens/puppet-filebeat/pull/281) +- Improve FreeBSD Support [\#282](https://github.com/pcfens/puppet-filebeat/pull/282) + +## [v4.9.0](https://github.com/pcfens/puppet-filebeat/tree/v4.9.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.8.0...v4.9.0) + +- Make pipeline available in all inputs [\#275](https://github.com/pcfens/puppet-filebeat/pull/275) +- Support autodiscovery [\#278](https://github.com/pcfens/puppet-filebeat/pull/278) +- Support `close_timeout` as a string [\#279](https://github.com/pcfens/puppet-filebeat/pull/279) + +## [v4.8.0](https://github.com/pcfens/puppet-filebeat/tree/v4.8.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.7.0...v4.8.0) + +- Update powershell dependency constraints [\#268](https://github.com/pcfens/puppet-filebeat/pull/268) +- Support ES cloud credentials [\267](https://github.com/pcfens/puppet-filebeat/pull/267) + +## [v4.7.0](https://github.com/pcfens/puppet-filebeat/tree/v4.7.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.6.0...v4.7.0) + +- Add `extra_validation_options` when validating the config [\#265](https://github.com/pcfens/puppet-filebeat/pull/265) + +## [v4.6.0](https://github.com/pcfens/puppet-filebeat/tree/v4.6.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.5.0...v4.6.0) + +- Allow processors to be applied to all inputs, and consolidated common configuration [\#260](https://github.com/pcfens/puppet-filebeat/pull/260) + +## [v4.5.0](https://github.com/pcfens/puppet-filebeat/tree/v4.5.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.4.2...v4.5.0) + +- Restart filebeat when config files are removed [\#258](https://github.com/pcfens/puppet-filebeat/pull/258) +- Support TCP and UDP input plugins [\#259](https://github.com/pcfens/puppet-filebeat/pull/259) + +## [v4.4.2](https://github.com/pcfens/puppet-filebeat/tree/v4.4.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.4.1...v4.4.2) + +- Fix empty monitoring parameter [\#257](https://github.com/pcfens/puppet-filebeat/issues/257) + +## [v4.4.1](https://github.com/pcfens/puppet-filebeat/tree/v4.4.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.4.0...v4.4.1) + +- Fix powershell module version dependency [\#256](https://github.com/pcfens/puppet-filebeat/pull/256 + +## [v4.4.0](https://github.com/pcfens/puppet-filebeat/tree/v4.4.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.3.1...v4.4.0) + +- Add the ability to supply the monitoring.* of the config file [\#252](https://github.com/pcfens/puppet-filebeat/issues/252) +- Add support for Windows Server 2016 and 2019 Core editions [\#255](https://github.com/pcfens/puppet-filebeat/pull/255) + +## [v4.3.1](https://github.com/pcfens/puppet-filebeat/tree/v4.3.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.3.0...v4.3.1) + +- Allow fields and tags to be defined for any input type [\#249](https://github.com/pcfens/puppet-filebeat/pull/249) + +## [v4.3.0](https://github.com/pcfens/puppet-filebeat/tree/v4.3.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.2.0...v4.3.0) + +- Support setting index index lifecycle policy [\#238](https://github.com/pcfens/puppet-filebeat/pull/238) +- Support logging overrides with Puppet older than 6.1 [\#241](https://github.com/pcfens/puppet-filebeat/pull/241) +- Allow inputs to be defined using just a hash (supports open ended inputs) [\#236](https://github.com/pcfens/puppet-filebeat/pull/236) + + +## [v4.2.0](https://github.com/pcfens/puppet-filebeat/tree/v4.2.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.1.2...v4.2.0) + +- Fix regression of processors type +- Add support for syslog inputs [\#232](https://github.com/pcfens/puppet-filebeat/pull/233) +- Upgrade PDK to 1.15.0 + +## [v4.1.2](https://github.com/pcfens/puppet-filebeat/tree/v4.1.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.1.1...v4.1.2) + +- Fix hardocded path to yum [\#229](https://github.com/pcfens/puppet-filebeat/pull/229) + +## [v4.1.1](https://github.com/pcfens/puppet-filebeat/tree/v4.1.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.1.0...v4.1.1) + +- Support old versions of filebeat for a version fact only [\#227](https://github.com/pcfens/puppet-filebeat/pull/227) +- Fix the processor input data type [\#228](https://github.com/pcfens/puppet-filebeat/issues/228) + +## [v4.1.0](https://github.com/pcfens/puppet-filebeat/tree/v4.1.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.5...v4.1.0) + +- Allow for override of the default systemd logging options [\#223](https://github.com/pcfens/puppet-filebeat/pull/223) + +## [v4.0.5](https://github.com/pcfens/puppet-filebeat/tree/v4.0.5) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.4...v4.0.5) + +- Update metadata for apt and stdlib modules [\#220](https://github.com/pcfens/puppet-filebeat/pull/220) +- Fix README typo [\#221](https://github.com/pcfens/puppet-filebeat/pull/221) + + +## [v4.0.4](https://github.com/pcfens/puppet-filebeat/tree/v4.0.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.3...v4.0.4) + +- Validate configuration in Windows [\#219](https://github.com/pcfens/puppet-filebeat/pull/219) + +## [v4.0.3](https://github.com/pcfens/puppet-filebeat/tree/v4.0.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.2...v4.0.3) + +- Fix `filebeat_version` fact on Windows [\#218](https://github.com/pcfens/puppet-filebeat/pull/218) + +## [v4.0.2](https://github.com/pcfens/puppet-filebeat/tree/v4.0.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.1...v4.0.2) + +- Clear the yum cache whenever we modify the yum repo config [\#217](https://github.com/pcfens/puppet-filebeat/issues/217) + +## [v4.0.1](https://github.com/pcfens/puppet-filebeat/tree/v4.0.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.0...v4.0.1) + +- Fix unit tests [\#216](https://github.com/pcfens/puppet-filebeat/pull/213) +- Fix fresh install regression [\#217](https://github.com/pcfens/puppet-filebeat/pull/216) + + +## [v4.0.0](https://github.com/pcfens/puppet-filebeat/tree/v4.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.4.0...v4.0.0) + +- Switch from `filebeat::prospector` to `filebeat::input` to reflect the changes + in the upstream filebeat configuration. +- Add support for Filebeat 7 +- Remove support for `registry_file` and `registry_flush` settings (removed in 7.x) +- Remove `queue_size` parameter + +## [v3.4.0](https://github.com/pcfens/puppet-filebeat/tree/v3.4.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.3...v3.4.0) + +- Add filebeat.config.modules section [\#204](https://github.com/pcfens/puppet-filebeat/pull/204) +- Fix filebeat::prospector::fields_under_root [\#205](https://github.com/pcfens/puppet-filebeat/pull/205) + +## [v3.3.3](https://github.com/pcfens/puppet-filebeat/tree/v3.3.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.2...v3.3.3) + +- Add a harvester limit [\#196](https://github.com/pcfens/puppet-filebeat/pull/196) +- Fix documentaion error [\#198](https://github.com/pcfens/puppet-filebeat/issues/198) +- Fix Puppet 4.10 undefined method error [\#199](https://github.com/pcfens/puppet-filebeat/pull/199) + +## [v3.3.2](https://github.com/pcfens/puppet-filebeat/tree/v3.3.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.1...v3.3.2) + +- Produce valid YAML for prospectors [\#193](https://github.com/pcfens/puppet-filebeat/pull/193) +- Upgrade to PDK 1.7.1 +- Add tests for Puppet 6 +- Add Puppet 6 support to metadata + +## [v3.3.1](https://github.com/pcfens/puppet-filebeat/tree/v3.3.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.0...v3.3.1) + +- Add a new `manage_apt` flag for disabling the inclusion of puppetlabs/apt [\#185](https://github.com/pcfens/puppet-filebeat/pull/185) + +## [v3.3.0](https://github.com/pcfens/puppet-filebeat/tree/v3.3.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.2.2...v3.3.0) + +- Allow use of puppet/archive 3.x [\#190](https://github.com/pcfens/puppet-filebeat/pull/190) +- Add support for Docker inputs [\#191](https://github.com/pcfens/puppet-filebeat/pull/191) +- Support puppetlabs/stdlib 5.x + +## [v3.2.2](https://github.com/pcfens/puppet-filebeat/tree/v3.2.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.2.1...v3.2.2) + +- Don't add xpack configuration when it's undef [\#187](https://github.com/pcfens/puppet-filebeat/pull/187) +- Don't disallow using puppetlabs/apt 6.x (check their [changelog](https://forge.puppet.com/puppetlabs/apt/changelog#600-2018-08-24) as this release drops support for Puppet pre 4.7) [\#186](https://github.com/pcfens/puppet-filebeat/pull/186) +- Use the latest PDK + +## [v3.2.1](https://github.com/pcfens/puppet-filebeat/tree/v3.2.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.2.0...v3.2.1) + +- Fetch the suse repository signing key over https [\#176](https://github.com/pcfens/puppet-filebeat/issues/176) + +## [v3.2.0](https://github.com/pcfens/puppet-filebeat/tree/v3.2.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.1.0...v3.2.0) + +- Add support for xpack monitoring [\#172](https://github.com/pcfens/puppet-filebeat/pull/172) +- Add support for OpenBSD [\#173](https://github.com/pcfens/puppet-filebeat/pull/173) +- Set filebeat_version to false when filebeat isn't installed [\#175](https://github.com/pcfens/puppet-filebeat/pull/175) + +## [v3.1.0](https://github.com/pcfens/puppet-filebeat/tree/v3.1.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.0.1...v3.1.0) + +- Manage filebeat modules as an array [\#168](https://github.com/pcfens/puppet-filebeat/pull/168) + +## [v3.0.1](https://github.com/pcfens/puppet-filebeat/tree/v3.0.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.0.0...v3.0.1) + +- Fix array validation in prospector defined resource [\#166](https://github.com/pcfens/puppet-filebeat/pull/166) + +## [v3.0.0](https://github.com/pcfens/puppet-filebeat/tree/v3.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.4.0...v3.0.0) + +Potentially Breaking Change: +- Make filebeat 6 the default version. + +Non-breaking changes: +- Allow setup entries in configuration [\#152](https://github.com/pcfens/puppet-filebeat/pull/152), [\#146](https://github.com/pcfens/puppet-filebeat/issues/146) +- Processors should be an array of hashes [\#157](https://github.com/pcfens/puppet-filebeat/pull/157), [\#156](https://github.com/pcfens/puppet-filebeat/issues/156) +- Validate URLs using stdlib [\#158](https://github.com/pcfens/puppet-filebeat/pull/158) +- Use external configuration setup for Filebeat 6+ [\#153](https://github.com/pcfens/puppet-filebeat/issues/153) +- Use version subcommand when determining version [\#159](https://github.com/pcfens/puppet-filebeat/issues/159) +- Add processors support to prospectors [\#162](https://github.com/pcfens/puppet-filebeat/pull/162) +- Update unsupported OS Family notice [\#161](https://github.com/pcfens/puppet-filebeat/pull/161) +- Use Puppet 4+ data types for prospectors [\#165](https://github.com/pcfens/puppet-filebeat/pull/165) +- Fix windows validation command [\#164](https://github.com/pcfens/puppet-filebeat/pull/164), [\#163](https://github.com/pcfens/puppet-filebeat/issues/163) ## [v2.4.0](https://github.com/pcfens/puppet-filebeat/tree/v2.4.0) [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.3.0...v2.4.0) diff --git a/Gemfile b/Gemfile index b44878f7..b67c93bb 100644 --- a/Gemfile +++ b/Gemfile @@ -1,119 +1,58 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org' def location_for(place_or_version, fake_version = nil) - if place_or_version =~ %r{\A(git[:@][^#]*)#(.*)} - [fake_version, { git: Regexp.last_match(1), branch: Regexp.last_match(2), require: false }].compact - elsif place_or_version =~ %r{\Afile:\/\/(.*)} - ['>= 0', { path: File.expand_path(Regexp.last_match(1)), require: false }] - else - [place_or_version, { require: false }] - end -end + git_url_regex = %r{\A(?(https?|git)[:@][^#]*)(#(?.*))?} + file_url_regex = %r{\Afile:\/\/(?.*)} -def gem_type(place_or_version) - if place_or_version =~ %r{\Agit[:@]} - :git - elsif !place_or_version.nil? && place_or_version.start_with?('file:') - :file + if place_or_version && (git_url = place_or_version.match(git_url_regex)) + [fake_version, { git: git_url[:url], branch: git_url[:branch], require: false }].compact + elsif place_or_version && (file_url = place_or_version.match(file_url_regex)) + ['>= 0', { path: File.expand_path(file_url[:path]), require: false }] else - :gem + [place_or_version, { require: false }] end end -ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments -minor_version = ruby_version_segments[0..1].join('.') - group :development do - gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') - gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') - gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') - gem "json", '= 1.8.1', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') - gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby] - gem "puppet-module-posix-dev-r#{minor_version}", require: false, platforms: [:ruby] - gem "puppet-module-win-default-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] - gem "puppet-module-win-dev-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "json", '= 2.1.0', require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "json", '= 2.3.0', require: false if Gem::Requirement.create(['>= 2.7.0', '< 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "json", '= 2.5.1', require: false if Gem::Requirement.create(['>= 3.0.0', '< 3.0.5']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "json", '= 2.6.1', require: false if Gem::Requirement.create(['>= 3.1.0', '< 3.1.3']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "json", '= 2.6.3', require: false if Gem::Requirement.create(['>= 3.2.0', '< 4.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "voxpupuli-puppet-lint-plugins", '~> 3.1', require: false + gem "facterdb", '~> 1.18', require: false + gem "metadata-json-lint", '>= 2.0.2', '< 4.0.0', require: false + gem "puppetlabs_spec_helper", '>= 3.0.0', '< 5.0.0', require: false + gem "rspec-puppet-facts", '~> 2.0', require: false + gem "codecov", '~> 0.2', require: false + gem "dependency_checker", '~> 0.2', require: false + gem "parallel_tests", '~> 3.4', require: false + gem "pry", '~> 0.10', require: false + gem "simplecov-console", '~> 0.5', require: false + gem "puppet-debugger", '~> 1.0', require: false + gem "rubocop", '= 1.19', require: false + gem "rubocop-performance", '= 1.9.1', require: false + gem "rubocop-rspec", '= 2.10.0', require: false + gem "rb-readline", '= 0.5.5', require: false, platforms: [:mswin, :mingw, :x64_mingw] end - group :system_tests do - gem "beaker", *location_for(ENV['BEAKER_VERSION'] || '>= 3') - gem "beaker-rspec", *location_for(ENV['BEAKER_RSPEC_VERSION']) - gem 'beaker-puppet_install_helper' - gem 'beaker-module_install_helper' + gem "puppet_litmus", '< 1.0.0', require: false, platforms: [:ruby] + gem "serverspec", '~> 2.41', require: false end puppet_version = ENV['PUPPET_GEM_VERSION'] -puppet_type = gem_type(puppet_version) facter_version = ENV['FACTER_GEM_VERSION'] hiera_version = ENV['HIERA_GEM_VERSION'] -def puppet_older_than?(version) - puppet_version = ENV['PUPPET_GEM_VERSION'] - !puppet_version.nil? && - Gem::Version.correct?(puppet_version) && - Gem::Requirement.new("< #{version}").satisfied_by?(Gem::Version.new(puppet_version.dup)) -end - gems = {} gems['puppet'] = location_for(puppet_version) # If facter or hiera versions have been specified via the environment -# variables, use those versions. If not, and if the puppet version is < 3.5.0, -# use known good versions of both for puppet < 3.5.0. -if facter_version - gems['facter'] = location_for(facter_version) -elsif puppet_type == :gem && puppet_older_than?('3.5.0') - gems['facter'] = ['>= 1.6.11', '<= 1.7.5', require: false] -end +# variables -if hiera_version - gems['hiera'] = location_for(ENV['HIERA_GEM_VERSION']) -elsif puppet_type == :gem && puppet_older_than?('3.5.0') - gem['hiera'] = ['>= 1.0.0', '<= 1.3.0', require: false] -end - -if Gem.win_platform? && (puppet_type != :gem || puppet_older_than?('3.5.0')) - # For Puppet gems < 3.5.0 (tested as far back as 3.0.0) on Windows - if puppet_type == :gem - gems['ffi'] = ['1.9.0', require: false] - gems['minitar'] = ['0.5.4', require: false] - gems['win32-eventlog'] = ['0.5.3', '<= 0.6.5', require: false] - gems['win32-process'] = ['0.6.5', '<= 0.7.5', require: false] - gems['win32-security'] = ['~> 0.1.2', '<= 0.2.5', require: false] - gems['win32-service'] = ['0.7.2', '<= 0.8.8', require: false] - else - gems['ffi'] = ['~> 1.9.0', require: false] - gems['minitar'] = ['~> 0.5.4', require: false] - gems['win32-eventlog'] = ['~> 0.5', '<= 0.6.5', require: false] - gems['win32-process'] = ['~> 0.6', '<= 0.7.5', require: false] - gems['win32-security'] = ['~> 0.1', '<= 0.2.5', require: false] - gems['win32-service'] = ['~> 0.7', '<= 0.8.8', require: false] - end - - gems['win32-dir'] = ['~> 0.3', '<= 0.4.9', require: false] - - if RUBY_VERSION.start_with?('1.') - gems['win32console'] = ['1.3.2', require: false] - # sys-admin was removed in Puppet 3.7.0 and doesn't compile under Ruby 2.x - gems['sys-admin'] = ['1.5.6', require: false] - end - - # Puppet < 3.7.0 requires these. - # Puppet >= 3.5.0 gem includes these as requirements. - # The following versions are tested to work with 3.0.0 <= puppet < 3.7.0. - gems['win32-api'] = ['1.4.8', require: false] - gems['win32-taskscheduler'] = ['0.2.2', require: false] - gems['windows-api'] = ['0.4.3', require: false] - gems['windows-pr'] = ['1.2.3', require: false] -elsif Gem.win_platform? - # If we're using a Puppet gem on Windows which handles its own win32-xxx gem - # dependencies (>= 3.5.0), set the maximum versions (see PUP-6445). - gems['win32-dir'] = ['<= 0.4.9', require: false] - gems['win32-eventlog'] = ['<= 0.6.5', require: false] - gems['win32-process'] = ['<= 0.7.5', require: false] - gems['win32-security'] = ['<= 0.2.5', require: false] - gems['win32-service'] = ['<= 0.8.8', require: false] -end +gems['facter'] = location_for(facter_version) if facter_version +gems['hiera'] = location_for(hiera_version) if hiera_version gems.each do |gem_name, gem_params| gem gem_name, *gem_params diff --git a/README.md b/README.md index 85df8d86..38adbda6 100644 --- a/README.md +++ b/README.md @@ -4,26 +4,44 @@ #### Table of Contents -1. [Description](#description) -2. [Setup - The basics of getting started with filebeat](#setup) +- [puppet-filebeat](#puppet-filebeat) + - [Table of Contents](#table-of-contents) + - [Description](#description) + - [Setup](#setup) - [What filebeat affects](#what-filebeat-affects) - - [Setup requirements](#setup-requirements) + - [Upgrading to Filebeat 7.x](#upgrading-to-filebeat-7x) + - [Setup Requirements](#setup-requirements) - [Beginning with filebeat](#beginning-with-filebeat) -3. [Usage - Configuration options and additional functionality](#usage) - - [Adding a prospector](#adding-a-prospector) + - [Usage](#usage) + - [Adding an Input](#adding-an-input) - [Multiline Logs](#multiline-logs) - - [JSON logs](#json-logs) - - [Prospectors in hiera](#prospectors-in-hiera) + - [JSON Logs](#json-logs) + - [Inputs in Hiera](#inputs-in-hiera) + - [Usage of filebeat modules](#usage-of-filebeat-modules) - [Usage on Windows](#usage-on-windows) - [Processors](#processors) -4. [Reference](#reference) + - [Processors in Hiera](#processors-in-hiera) + - [Index Lifecycle Management](#index-lifecycle-management) + - [Reference](#reference) - [Public Classes](#public-classes) + - [Class: `filebeat`](#class-filebeat) - [Private Classes](#private-classes) + - [Class: `filebeat::config`](#class-filebeatconfig) + - [Class: `filebeat::install`](#class-filebeatinstall) + - [Class: `filebeat::params`](#class-filebeatparams) + - [Class: `filebeat::repo`](#class-filebeatrepo) + - [Class: `filebeat::service`](#class-filebeatservice) + - [Class: `filebeat::install::linux`](#class-filebeatinstalllinux) + - [Class: `filebeat::install::windows`](#class-filebeatinstallwindows) - [Public Defines](#public-defines) -5. [Limitations - OS compatibility, etc.](#limitations) - - [Pre-1.9.1 Ruby](#pre-191-ruby) + - [Define: `filebeat::input`](#define-filebeatinput) + - [Define: `filebeat::module`](#define-filebeatmodule) + - [Limitations](#limitations) + - [Generic template](#generic-template) + - [Debian Systems](#debian-systems) - [Using config_file](#using-config_file) -6. [Development - Guide for contributing to the module](#development) + - [Logging on systems with Systemd and with version filebeat 7.0+ installed](#logging-on-systems-with-systemd-and-with-version-filebeat-70-installed) + - [Development](#development) ## Description @@ -36,9 +54,12 @@ The `filebeat` module installs and configures the [filebeat log shipper](https:/ By default `filebeat` adds a software repository to your system, and installs filebeat along with required configurations. -### Upgrading to Filebeat 6.x +### Upgrading to Filebeat 7.x -To upgrade to Filebeat 6.x, simply set `$filebeat::major_version` to `6` and `$filebeat::package_ensure` to `latest`. +To upgrade to Filebeat 7.x, simply set `$filebeat::major_version` to `7` and `$filebeat::package_ensure` to `latest` (or whichever version of 7.x you want, just not present). + +You'll also need to change instances of `filebeat::prospector` to `filebeat::input` when upgrading to version 4.x of +this module. ### Setup Requirements @@ -66,7 +87,6 @@ class { 'filebeat': 'http://anotherserver:9200' ], 'loadbalance' => true, - 'index' => 'packetbeat', 'cas' => [ '/etc/pki/root/ca.pem', ], @@ -96,18 +116,18 @@ class { 'filebeat': [logging](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-logging) options can be configured the same way, and are documented on the [elastic website](https://www.elastic.co/guide/en/beats/filebeat/current/index.html). -### Adding a prospector +### Adding an Input -Prospectors are processes that ship log files to elasticsearch or logstash. They can +Inputs are processes that ship log files to elasticsearch or logstash. They can be defined as a hash added to the class declaration (also used for automatically creating -prospectors using hiera), or as their own defined resources. +input using hiera), or as their own defined resources. At a minimum, the `paths` parameter must be set to an array of files or blobs that should be shipped. `doc_type` is what logstash views as the type parameter if you'd like to apply conditional filters. ```puppet -filebeat::prospector { 'syslogs': +filebeat::input { 'syslogs': paths => [ '/var/log/auth.log', '/var/log/syslog', @@ -118,23 +138,58 @@ filebeat::prospector { 'syslogs': #### Multiline Logs -Filebeat prospectors can handle multiline log entries. The `multiline` +Filebeat inputs can handle multiline log entries. The `multiline` parameter accepts a hash containing `pattern`, `negate`, `match`, `max_lines`, and `timeout` as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html). #### JSON Logs -Filebeat prospectors (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json` -parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key` -as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/5.5/configuration-filebeat-options.html#config-json). +Filebeat inputs (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json` +parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key`. + +Depending on the version, `expand_keys`, `document_id` and `ignore_decoding_error` may be supported as well. -### Prospectors in Hiera +See the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/7.11/filebeat-input-log.html#filebeat-input-log-config-json) for details. -Prospectors can be defined in hiera using the `prospectors` parameter. By default, hiera will not merge -prospector declarations down the hiera hierarchy. That behavior can be changed by configuring the +### Inputs in Hiera + +Inputs can be defined in hiera using the `inputs` parameter. By default, hiera will not merge +input declarations down the hiera hierarchy. That behavior can be changed by configuring the [lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) flag. +`inputs` can be a Hash that will follow all the parameters listed on this documentation or an +Array that will output as is to the input config file. + +### Usage of filebeat modules + +Filebeat ships with modules which contain pipelines and dashboards for common software. Filebeat needs to be setup to ship directly into elasticsearch that +it's possible that filebeat will setup pipelines and dashboards automatically. + +If your setup includes logstash or some other service between filebeat and elasticsearch the following settings might not work as expected. + +The following should be a minimal example to get `filebeat::module::*` to create the required config and push pipeline and dashboards into your elasticsearch & kibana. + +```puppet +class { 'filebeat::module::system': + syslog_enabled => true, + auth_enabled => true, +} + +class { 'filebeat': + enable_conf_modules => true, + overwrite_pipelines => true, + setup => { + dashboards => { + enabled => true + }, + kibana => { + host => 'http://kibana.example.com:5601', + } + } +} +``` + ### Usage on Windows When installing on Windows, this module will download the windows version of Filebeat from @@ -152,41 +207,49 @@ processors using hiera), or as their own defined resources. To drop the offset and input_type fields from all events: ```puppet -class{"filebeat": - processors => { - "drop_fields" => { - "params" => {"fields" => ["input_type", "offset"]} - }, - }, +class {'filebeat': + processors => [ + { + 'drop_fields' => { + 'fields' => ['input_type', 'offset'], + } + } + ], } ``` To drop all events that have the http response code equal to 200: - +input ```puppet -class{"filebeat": - processors => { - "drop_event" => { - "when" => {"equals" => {"http.code" => 200}} - }, - }, +class {'filebeat': + processors => [ + { + 'drop_event' => { + 'when' => {'equals' => {'http.code' => 200}} + } + } + ], } ``` Now to combine these examples into a single definition: ```puppet -class{"filebeat": - processors => { - "drop_fields" => { - "params" => {"fields" => ["input_type", "offset"]}, - "priority" => 1, - }, - "drop_event" => { - "when" => {"equals" => {"http.code" => 200}}, - "priority: => 2, +class {'filebeat': + processors => [ + { + 'drop_fields' => { + 'params' => {'fields' => ['input_type', 'offset']}, + 'priority' => 1, + } }, - }, + { + 'drop_event' => { + 'when' => {'equals' => {'http.code' => 200}}, + 'priority' => 2, + } + } + ], } ``` @@ -199,20 +262,60 @@ processor declarations down the hiera hierarchy. That behavior can be changed by [lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) flag. +### Index Lifecycle Management + +You can override the default filebeat ILM policy by specifying `ilm.policy` hash in `filebeat::setup` parameter: + +``` +filebeat::setup: + ilm.policy: + phases: + hot: + min_age: "0ms" + actions: + rollover: + max_size: "10gb" + max_age: "1d" +``` + ## Reference - - [**Public Classes**](#public-classes) - - [Class: filebeat](#class-filebeat) - - [**Private Classes**](#private-classes) - - [Class: filebeat::config](#class-filebeatconfig) - - [Class: filebeat::install](#class-filebeatinstall) - - [Class: filebeat::params](#class-filebeatparams) - - [Class: filebeat::repo](#class-filebeatrepo) - - [Class: filebeat::service](#class-filebeatservice) - - [Class: filebeat::install::linux](#class-filebeatinstalllinux) - - [Class: filebeat::install::windows](#class-filebeatinstallwindows) - - [**Public Defines**](#public-defines) - - [Define: filebeat::prospector](#define-filebeatprospector) - - [Define: filebeat::processors](#define-filebeatprocessor) +- [puppet-filebeat](#puppet-filebeat) + - [Table of Contents](#table-of-contents) + - [Description](#description) + - [Setup](#setup) + - [What filebeat affects](#what-filebeat-affects) + - [Upgrading to Filebeat 7.x](#upgrading-to-filebeat-7x) + - [Setup Requirements](#setup-requirements) + - [Beginning with filebeat](#beginning-with-filebeat) + - [Usage](#usage) + - [Adding an Input](#adding-an-input) + - [Multiline Logs](#multiline-logs) + - [JSON Logs](#json-logs) + - [Inputs in Hiera](#inputs-in-hiera) + - [Usage on Windows](#usage-on-windows) + - [Processors](#processors) + - [Processors in Hiera](#processors-in-hiera) + - [Index Lifecycle Management](#index-lifecycle-management) + - [Reference](#reference) + - [Public Classes](#public-classes) + - [Class: `filebeat`](#class-filebeat) + - [Private Classes](#private-classes) + - [Class: `filebeat::config`](#class-filebeatconfig) + - [Class: `filebeat::install`](#class-filebeatinstall) + - [Class: `filebeat::params`](#class-filebeatparams) + - [Class: `filebeat::repo`](#class-filebeatrepo) + - [Class: `filebeat::service`](#class-filebeatservice) + - [Class: `filebeat::install::linux`](#class-filebeatinstalllinux) + - [Class: `filebeat::install::windows`](#class-filebeatinstallwindows) + - [Public Defines](#public-defines) + - [Define: `filebeat::input`](#define-filebeatinput) + - [Define: `filebeat::module`](#define-filebeatmodule) + - [Limitations](#limitations) + - [Generic template](#generic-template) + - [Debian Systems](#debian-systems) + - [Using config\_file](#using-config_file) + - [Logging on systems with Systemd and with version filebeat 7.0+ installed](#logging-on-systems-with-systemd-and-with-version-filebeat-70-installed) + - [Development](#development) ### Public Classes @@ -222,52 +325,65 @@ Installs and configures filebeat. **Parameters within `filebeat`** - `package_ensure`: [String] The ensure parameter for the filebeat package If set to absent, - prospectors and processors passed as parameters are ignored and everything managed by + inputs and processors passed as parameters are ignored and everything managed by puppet will be removed. (default: present) +- `manage_package`: [Boolean] Whether ot not to manage the installation of the package (default: true) - `manage_repo`: [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) -- `major_version`: [Enum] The major version of Filebeat to install. Should be either `5` or `6`. The default value is `5`. +- `major_version`: [Enum] The major version of Filebeat to install. Should be either `'5'` or `'6'`. The default value is `'6'`, except + for OpenBSD 6.3 and earlier, which has a default value of `'5'`. - `service_ensure`: [String] The ensure parameter on the filebeat service (default: running) - `service_enable`: [String] The enable parameter on the filebeat service (default: true) - `param repo_priority`: [Integer] Repository priority. yum and apt supported (default: undef) - `service_provider`: [String] The provider parameter on the filebeat service (default: on RedHat based systems use redhat, otherwise undefined) - `spool_size`: [Integer] How large the spool should grow before being flushed to the network (default: 2048) - `idle_timeout`: [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) -- `publish_async`: [Boolean] If set to true filebeat will publish while preparing the next batch of lines to transmit (defualt: false) -- `registry_file`: [String] The registry file used to store positions, must be an absolute path (default is OS dependent - see params.pp) +- `publish_async`: [Boolean] If set to true filebeat will publish while preparing the next batch of lines to transmit (default: false) - `config_file`: [String] Where the configuration file managed by this module should be placed. If you think you might want to use this, read the [limitations](#using-config_file) first. Defaults to the location that filebeat expects for your operating system. -- `config_dir`: [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) +- `config_dir`: [String] The directory where inputs should be defined (default: /etc/filebeat/conf.d) - `config_dir_mode`: [String] The permissions mode set on the configuration directory (default: 0755) - `config_dir_owner`: [String] The owner of the configuration directory (default: root). Linux only. - `config_dir_group`: [String] The group of the configuration directory (default: root). Linux only. - `config_file_mode`: [String] The permissions mode set on configuration files (default: 0644) -- `config_file_owner`: [String] The owner of the configuration files, including prospectors (default: root). Linux only. -- `config_file_group`: [String] The group of the configuration files, including prospectors (default: root). Linux only. -- `purge_conf_dir`: [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged +- `config_file_owner`: [String] The owner of the configuration files, including inputs (default: root). Linux only. +- `config_file_group`: [String] The group of the configuration files, including inputs (default: root). Linux only. +- `purge_conf_dir`: [Boolean] Should files in the input configuration directory not managed by puppet be automatically purged +- `enable_conf_modules`: [Boolean] Should filebeat.config.modules be enabled +- `modules_dir`: [String] The directory where module configurations should be defined (default: /etc/filebeat/modules.d) +- `cloud`: [Hash] Will be converted to YAML for the optional cloud.id and cloud.auth of the configuration (see documentation, and above) +- `features`: [Hash] Will be converted to YAML for the optional features section of the configuration (see documentation, and above) +- `queue`: [Hash] Will be converted to YAML for the optional queue.mem and queue.disk of the configuration (see documentation, and above) - `outputs`: [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) - `shipper`: [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) +- `autodiscover`: [Hash] Will be converted to YAML for the optional autodiscover section of the configuration (see documentation, and above)` - `logging`: [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) +- `systemd_beat_log_opts_override`: [String] Will overide the default `BEAT_LOG_OPTS=-e`. Required if using `logging` hash on systems running with systemd. required: Puppet 6.1+, Filebeat 7+, +- `modules`: [Array] Will be converted to YAML to create the optional modules section of the filebeat config (see documentation) - `conf_template`: [String] The configuration template to use to generate the main filebeat.yml config file. - `download_url`: [String] The URL of the zip file that should be downloaded to install filebeat (windows only) - `install_dir`: [String] Where filebeat should be installed (windows only) - `tmp_dir`: [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only) - `shutdown_timeout`: [String] How long filebeat waits on shutdown for the publisher to finish sending events -- `beat_name`: [String] The name of the beat shipper (default: hostname) +- `beat_name`: [String] The name of the beat shipper (default: FQDN) - `tags`: [Array] A list of tags that will be included with each published transaction -- `queue_size`: [String] The internal queue size for events in the pipeline - `max_procs`: [Number] The maximum number of CPUs that can be simultaneously used - `fields`: [Hash] Optional fields that should be added to each event output - `fields_under_root`: [Boolean] If set to true, custom fields are stored in the top level instead of under fields - `disable_config_test`: [Boolean] If set to true, configuration tests won't be run on config files before writing them. -- `processors`: [Hash] Processors that should be configured. -- `prospectors`: [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera +- `processors`: [Array] Processors that should be configured. +- `monitoring`: [Hash] The monitoring.* components of the filebeat configuration. +- `inputs`: [Hash] or [Array] Inputs that will be created. Commonly used to create inputs using hiera +- `setup`: [Hash] Setup that will be created. Commonly used to create setup using hiera +- `xpack`: [Hash] XPack configuration to pass to filebeat +- `extra_validate_options`: [String] Extra command line options to pass to the configuration validation command. +- `overwrite_pipelines`: [Boolean] If set to true, filebeat will overwrite existing pipelines. ### Private Classes #### Class: `filebeat::config` -Creates the configuration files required for filebeat (but not the prospectors) +Creates the configuration files required for filebeat (but not the inputs) #### Class: `filebeat::install` @@ -295,19 +411,27 @@ Downloads, extracts, and installs the filebeat zip file in Windows. ### Public Defines -#### Define: `filebeat::prospector` +#### Define: `filebeat::input` -Installs a configuration file for a prospector. +Installs a configuration file for a input. Be sure to read the [filebeat configuration details](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html) to fully understand what these parameters do. -**Parameters for `filebeat::prospector`** - - `ensure`: The ensure parameter on the prospector configuration file. (default: present) - - `paths`: [Array] The paths, or blobs that should be handled by the prospector. (required) +**Parameters for `filebeat::input`** + - `ensure`: The ensure parameter on the input configuration file. (default: present) + - `paths`: [Array] The paths, or blobs that should be handled by the input. (required if input_type is _log_) + - `containers_ids`: [Array] If input_type is _docker_, the list of Docker container ids to read the logs from. (default: '*') + - `containers_path`: [String] If input_type is _docker_, the path from where the logs should be read from. (default: /var/log/docker/containers) + - `containers_stream`: [String] If input_type is _docker_, read from the specified stream only. (default: all) + - `combine_partial`: [Boolean] If input_type is _docker_, enable partial messages joining. (default: false) + - `cri_parse_flags`: [Boolean] If input_type is _docker_, enable CRI flags parsing from the log file. (default: false) + - `syslog_protocol`: [Enum tcp,udp] Syslog protocol (default: udp) + - `syslog_host`: [String] Host to listen for syslog messages (default: localhost:5140) - `exclude_files`: [Array] Files that match any regex in the list are excluded from filebeat (default: []) - `encoding`: [String] The file encoding. (default: plain) - - `input_type`: [String] log or stdin - where filebeat reads the log from (default:log) + - `input_type`: [String] where filebeat reads the log from (default: filestream) + - `take_over` : [Boolean] Optionally enable [`take_over`](https://www.elastic.co/guide/en/beats/filebeat/8.11/filebeat-input-filestream.html#filebeat-input-filestream-take-over) when switchting from the deprecated input type `log` to the new input type `filestream`. This avoids re-ingesting already logfiles Filebeat already read when switching to `filestream`. This feature requires Filebeat 8.x. - `fields`: [Hash] Optional fields to add information to the output (default: {}) - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents. - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing) @@ -316,7 +440,7 @@ to fully understand what these parameters do. - `log_type`: [String] \(Deprecated - use `doc_type`\) The document_type setting (optional - default: log) - `doc_type`: [String] The event type to used for published lines, used as type field in logstash and elasticsearch (optional - default: log) - - `scan_frequency`: [String] How often should the prospector check for new files (default: 10s) + - `scan_frequency`: [String] How often should the input check for new files (default: 10s) - `harvester_buffer_size`: [Integer] The buffer size the harvester uses when fetching the file (default: 16384) - `tail_files`: [Boolean] If true, filebeat starts reading new files at the end instead of the beginning (default: false) - `backoff`: [String] How long filebeat should wait between scanning a file after reaching EOF (default: 1s) @@ -324,16 +448,32 @@ to fully understand what these parameters do. - `backoff_factor`: [Integer] `backoff` is multiplied by this parameter until `max_backoff` is reached to determine the actual backoff (default: 2) - `force_close_files`: [Boolean] Should filebeat forcibly close a file when renamed (default: false) - - `pipeline`: [String] Filebeat can be configured for a different ingest pipeline for each prospector (default: undef) + - `pipeline`: [String] Filebeat can be configured for a different ingest pipeline for each input (default: undef) - `include_lines`: [Array] A list of regular expressions to match the lines that you want to include. Ignored if empty (default: []) - `exclude_lines`: [Array] A list of regular expressions to match the files that you want to exclude. Ignored if empty (default: []) - `max_bytes`: [Integer] The maximum number of bytes that a single log message can have (default: 10485760) + - `tags`: [Array] A list of tags to send along with the log data. - `json`: [Hash] Options that control how filebeat handles decoding of log messages in JSON format [See above](#json-logs). (default: {}) - `multiline`: [Hash] Options that control how Filebeat handles log messages that span multiple lines. [See above](#multiline-logs). (default: {}) + - `host`: [String] Host and port used to read events for TCP or UDP plugin (default: localhost:9000) + - `max_message_size`: [String] The maximum size of the message received over TCP or UDP (default: undef) + - `keep_null`: [Boolean] If this option is set to true, fields with null values will be published in the output document (default: undef) + - `include_matches`: [Array] Journald input only, A collection of filter expressions used to match fields. The format of the expression is field=value (default: []) + - `seek`: [Enum] Journald input only, The position to start reading the journal from (default: undef) + - `index`: [String] If present, this formatted string overrides the index for events from this input (for elasticsearch outputs), or sets the raw_index field of the event’s metadata (for other outputs) (default: undef) + - `publisher_pipeline_disable_host`: [Boolean] This disables the "host.name" attribute being added to events. See [filebeat input configuration reference](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#_publisher_pipeline_disable_host_13) (default: false) + +#### Define: `filebeat::module` + +Base resource used to implement filebeat module support in this puppet module and can be useful if you have custom filebeat modules. + +**Parameters for `filebeat::module`** + - `ensure`: The ensure parameter on the module configuration file. (default: present) + - `config`: [Hash] Full hash representation of the module configuration ## Limitations This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping @@ -349,15 +489,6 @@ By default, a generic, open ended template is used that simply converts your con a hash that is produced as YAML on the system. To use a template that is more strict, but possibly incomplete, set `conf_template` to `filebeat/filebeat.yml.erb`. -### Registry Path - -The default registry file in this module doesn't match the filebeat default, but moving the file -while the filbeat service is running can cause data duplication or data loss. If you're installing -filebeat for the first time you should consider setting `registry_file` to match the -[default](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-global-options.html#_registry_file). - -Be sure to include a path or the file will be put at the root of your filesystem. - ### Debian Systems Filebeat 5.x and newer requires apt-transport-https, but this module won't install it for you. @@ -381,6 +512,45 @@ file { '/etc/filebeat/filebeat.yml': ``` to ensure that services are managed like you might expect. +### Logging on systems with Systemd and with version filebeat 7.0+ installed +With filebeat version 7+ running on systems with systemd, the filebeat systemd service file contains a default that will ignore the logging hash parameter + +``` +Environment="BEAT_LOG_OPTS=-e` +``` +to overide this default, you will need to set the systemd_beat_log_opts_override parameter to empty string + +example: +```puppet +class {'filebeat': + logging => { + 'level' => 'debug', + 'to_syslog' => false, + 'to_files' => true, + 'files' => { + 'path' => '/var/log/filebeat', + 'name' => 'filebeat', + 'keepfiles' => '7', + 'permissions' => '0644' + }, + systemd_beat_log_opts_override => "", +} +``` + +this will only work on systems with puppet version 6.1+. On systems with puppet version < 6.1 you will need to `systemctl daemon-reload`. This can be achived by using the [camptocamp-systemd](https://forge.puppet.com/camptocamp/systemd) + +```puppet +include systemd::systemctl::daemon_reload + +class {'filebeat': + logging => { +... + }, + systemd_beat_log_opts_override => "", + notify => Class['systemd::systemctl::daemon_reload'], +} +``` + ## Development Pull requests and bug reports are welcome. If you're sending a pull request, please consider diff --git a/Rakefile b/Rakefile index 81381e0c..0f8754eb 100644 --- a/Rakefile +++ b/Rakefile @@ -1,2 +1,89 @@ +# frozen_string_literal: true + +require 'bundler' +require 'puppet_litmus/rake_tasks' if Bundler.rubygems.find_name('puppet_litmus').any? require 'puppetlabs_spec_helper/rake_tasks' require 'puppet-syntax/tasks/puppet-syntax' +require 'puppet_blacksmith/rake_tasks' if Bundler.rubygems.find_name('puppet-blacksmith').any? +require 'github_changelog_generator/task' if Bundler.rubygems.find_name('github_changelog_generator').any? +require 'puppet-strings/tasks' if Bundler.rubygems.find_name('puppet-strings').any? + +def changelog_user + return unless Rake.application.top_level_tasks.include? "changelog" + returnVal = nil || JSON.load(File.read('metadata.json'))['author'] + raise "unable to find the changelog_user in .sync.yml, or the author in metadata.json" if returnVal.nil? + puts "GitHubChangelogGenerator user:#{returnVal}" + returnVal +end + +def changelog_project + return unless Rake.application.top_level_tasks.include? "changelog" + + returnVal = nil + returnVal ||= begin + metadata_source = JSON.load(File.read('metadata.json'))['source'] + metadata_source_match = metadata_source && metadata_source.match(%r{.*\/([^\/]*?)(?:\.git)?\Z}) + + metadata_source_match && metadata_source_match[1] + end + + raise "unable to find the changelog_project in .sync.yml or calculate it from the source in metadata.json" if returnVal.nil? + + puts "GitHubChangelogGenerator project:#{returnVal}" + returnVal +end + +def changelog_future_release + return unless Rake.application.top_level_tasks.include? "changelog" + returnVal = "v%s" % JSON.load(File.read('metadata.json'))['version'] + raise "unable to find the future_release (version) in metadata.json" if returnVal.nil? + puts "GitHubChangelogGenerator future_release:#{returnVal}" + returnVal +end + +PuppetLint.configuration.send('disable_relative') + + +if Bundler.rubygems.find_name('github_changelog_generator').any? + GitHubChangelogGenerator::RakeTask.new :changelog do |config| + raise "Set CHANGELOG_GITHUB_TOKEN environment variable eg 'export CHANGELOG_GITHUB_TOKEN=valid_token_here'" if Rake.application.top_level_tasks.include? "changelog" and ENV['CHANGELOG_GITHUB_TOKEN'].nil? + config.user = "#{changelog_user}" + config.project = "#{changelog_project}" + config.future_release = "#{changelog_future_release}" + config.exclude_labels = ['maintenance'] + config.header = "# Change log\n\nAll notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org)." + config.add_pr_wo_labels = true + config.issues = false + config.merge_prefix = "### UNCATEGORIZED PRS; LABEL THEM ON GITHUB" + config.configure_sections = { + "Changed" => { + "prefix" => "### Changed", + "labels" => ["backwards-incompatible"], + }, + "Added" => { + "prefix" => "### Added", + "labels" => ["enhancement", "feature"], + }, + "Fixed" => { + "prefix" => "### Fixed", + "labels" => ["bug", "documentation", "bugfix"], + }, + } + end +else + desc 'Generate a Changelog from GitHub' + task :changelog do + raise < 1.15' + condition: "Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.3.0')" +EOM + end +end + diff --git a/appveyor.yml b/appveyor.yml index bb7cc46f..a70c01eb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,10 @@ +--- version: 1.1.x.{build} +skip_branch_with_pr: true +branches: + only: + - main + - release skip_commits: message: /^\(?doc\)?.*/ clone_depth: 10 @@ -9,38 +15,22 @@ init: - 'mkdir C:\ProgramData\PuppetLabs\hiera && exit 0' - 'mkdir C:\ProgramData\PuppetLabs\puppet\var && exit 0' environment: - PUPPET_GEM_VERSION: ~> 4.0 matrix: - - RUBY_VERSION: 24-x64 - CHECK: syntax lint + RUBY_VERSION: 25-x64 + CHECK: validate lint check rubocop - - RUBY_VERSION: 24-x64 - CHECK: metadata_lint + PUPPET_GEM_VERSION: ~> 6.0 + RUBY_VERSION: 25 + CHECK: parallel_spec - - RUBY_VERSION: 24-x64 - CHECK: rubocop - - - RUBY_VERSION: 24-x64 - CHECK: spec - - - RUBY_VERSION: 21-x64 - CHECK: spec + PUPPET_GEM_VERSION: ~> 6.0 + RUBY_VERSION: 25-x64 + CHECK: parallel_spec matrix: fast_finish: true install: - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH% - # Due to a bug in the version of OpenSSL shipped with Ruby 2.4.1 on Windows - # (https://bugs.ruby-lang.org/issues/11033). Errors are ignored because the - # mingw gem calls out to pacman to install OpenSSL which is already - # installed, causing gem to raise a warning that powershell determines to be - # a fatal error. - - ps: | - $ErrorActionPreference = "SilentlyContinue" - if($env:RUBY_VERSION -eq "24-x64") { - gem install openssl "~> 2.0.4" --no-rdoc --no-ri -- --with-openssl-dir=C:\msys64\mingw64 - } - $host.SetShouldExit(0) - bundle install --jobs 4 --retry 2 --without system_tests - type Gemfile.lock build: off diff --git a/lib/facter/filebeat_version.rb b/lib/facter/filebeat_version.rb index 2a0a6df2..bbeb8215 100644 --- a/lib/facter/filebeat_version.rb +++ b/lib/facter/filebeat_version.rb @@ -1,16 +1,35 @@ require 'facter' Facter.add('filebeat_version') do - confine 'kernel' => %w[FreeBSD Linux Windows] + confine 'kernel' => ['FreeBSD', 'OpenBSD', 'Linux', 'Windows', 'SunOS'] if File.executable?('/usr/bin/filebeat') - filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat --version') + filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat version') + if filebeat_version.empty? or filebeat_version.include? 'error' + filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat --version') + end + elsif File.executable?('/usr/local/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat version') + if filebeat_version.empty? or filebeat_version.include? 'error' + filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat --version') + end + elsif File.executable?('/opt/local/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/opt/local/bin/filebeat version') + if filebeat_version.empty? or filebeat_version.include? 'error' + filebeat_version = Facter::Util::Resolution.exec('/opt/local/bin/filebeat --version') + end elsif File.executable?('/usr/share/filebeat/bin/filebeat') filebeat_version = Facter::Util::Resolution.exec('/usr/share/filebeat/bin/filebeat --version') elsif File.executable?('/usr/local/sbin/filebeat') - filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat --version') + filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat version') + if filebeat_version.empty? or filebeat_version.include? 'error' + filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat --version') + end elsif File.exist?('c:\Program Files\Filebeat\filebeat.exe') - filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version') + filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" version') + if filebeat_version.empty? or filebeat_version.include? 'error' + filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version') + end end setcode do - %r{^filebeat version ([^\s]+)?}.match(filebeat_version)[1] unless filebeat_version.nil? + filebeat_version.nil? ? false : %r{^filebeat version ([^\s]+)?}.match(filebeat_version)[1] end end diff --git a/manifests/config.pp b/manifests/config.pp index 79cca198..c623075d 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -6,63 +6,121 @@ class filebeat::config { $major_version = $filebeat::major_version + if 'ilm.policy' in $filebeat::setup { + file { "${filebeat::config_dir}/ilm_policy.json": + content => to_json({ 'policy' => $filebeat::setup['ilm.policy'] }), + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + $setup = $filebeat::setup - 'ilm.policy' + { 'ilm.policy_file' => "${filebeat::config_dir}/ilm_policy.json" } + } else { + $setup = $filebeat::setup + } + if versioncmp($major_version, '6') >= 0 { - $filebeat_config = delete_undef_values({ - 'shutdown_timeout' => $filebeat::shutdown_timeout, - 'name' => $filebeat::beat_name, - 'tags' => $filebeat::tags, - 'max_procs' => $filebeat::max_procs, - 'fields' => $filebeat::fields, - 'fields_under_root' => $filebeat::fields_under_root, - 'filebeat' => { - 'registry_file' => $filebeat::registry_file, - 'config_dir' => $filebeat::config_dir, - 'shutdown_timeout' => $filebeat::shutdown_timeout, - }, - 'output' => $filebeat::outputs, - 'shipper' => $filebeat::shipper, - 'logging' => $filebeat::logging, - 'runoptions' => $filebeat::run_options, - 'processors' => $filebeat::processors, + $filebeat_config_temp = delete_undef_values({ + 'name' => $filebeat::beat_name, + 'tags' => $filebeat::tags, + 'max_procs' => $filebeat::max_procs, + 'fields' => $filebeat::fields, + 'fields_under_root' => $filebeat::fields_under_root, + 'filebeat' => { + 'config.inputs' => { + 'enabled' => true, + 'path' => "${filebeat::config_dir}/*.yml", + }, + 'config.modules' => { + 'enabled' => $filebeat::enable_conf_modules, + 'path' => "${filebeat::modules_dir}/*.yml", + }, + 'modules' => $filebeat::modules, + 'overwrite_pipelines' => $filebeat::overwrite_pipelines, + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'registry' => { + 'path' => $filebeat::registry_path, + 'file_permissions' => $filebeat::registry_file_permissions, + 'flush' => $filebeat::registry_flush, + }, + 'autodiscover' => $filebeat::autodiscover, + }, + 'http' => $filebeat::http, + 'cloud' => $filebeat::cloud, + 'queue' => $filebeat::queue, + 'output' => $filebeat::outputs, + 'shipper' => $filebeat::shipper, + 'logging' => $filebeat::logging, + 'runoptions' => $filebeat::run_options, + 'processors' => $filebeat::processors, + 'monitoring' => $filebeat::monitoring, + 'setup' => $setup, }) + # Add the 'xpack' section if supported (version >= 6.1.0) and not undef + if $filebeat::xpack and versioncmp($filebeat::package_ensure, '6.1.0') >= 0 { + $filebeat_config_xpack = deep_merge($filebeat_config_temp, { 'xpack' => $filebeat::xpack }) + } + else { + $filebeat_config_xpack = $filebeat_config_temp + } + # Add the 'features' section if supported (version >= 8.7.0) and not undef + if $filebeat::features and versioncmp($filebeat::package_ensure, '8.7.0') >= 0 { + $filebeat_config = deep_merge($filebeat_config_xpack, { 'features' => $filebeat::features }) + } + else { + $filebeat_config = $filebeat_config_xpack + } } else { - $filebeat_config = delete_undef_values({ - 'shutdown_timeout' => $filebeat::shutdown_timeout, - 'name' => $filebeat::beat_name, - 'tags' => $filebeat::tags, - 'queue_size' => $filebeat::queue_size, - 'max_procs' => $filebeat::max_procs, - 'fields' => $filebeat::fields, - 'fields_under_root' => $filebeat::fields_under_root, - 'filebeat' => { - 'spool_size' => $filebeat::spool_size, - 'idle_timeout' => $filebeat::idle_timeout, - 'registry_file' => $filebeat::registry_file, - 'publish_async' => $filebeat::publish_async, - 'config_dir' => $filebeat::config_dir, - 'shutdown_timeout' => $filebeat::shutdown_timeout, - }, - 'output' => $filebeat::outputs, - 'shipper' => $filebeat::shipper, - 'logging' => $filebeat::logging, - 'runoptions' => $filebeat::run_options, - 'processors' => $filebeat::processors, + $filebeat_config_temp = delete_undef_values({ + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'name' => $filebeat::beat_name, + 'tags' => $filebeat::tags, + 'queue_size' => $filebeat::queue_size, + 'max_procs' => $filebeat::max_procs, + 'fields' => $filebeat::fields, + 'fields_under_root' => $filebeat::fields_under_root, + 'filebeat' => { + 'config_dir' => $filebeat::config_dir, + 'idle_timeout' => $filebeat::idle_timeout, + 'overwrite_pipelines' => $filebeat::overwrite_pipelines, + 'publish_async' => $filebeat::publish_async, + 'registry_file' => $filebeat::registry_file, + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'spool_size' => $filebeat::spool_size, + }, + 'output' => $filebeat::outputs, + 'shipper' => $filebeat::shipper, + 'logging' => $filebeat::logging, + 'runoptions' => $filebeat::run_options, + 'processors' => $filebeat::processors, }) + # Add the 'modules' section if supported (version >= 5.2.0) + if versioncmp($filebeat::package_ensure, '5.2.0') >= 0 { + $filebeat_config = deep_merge($filebeat_config_temp, { 'modules' => $filebeat::modules }) + } + else { + $filebeat_config = $filebeat_config_temp + } } - Filebeat::Prospector <| |> -> File['filebeat.yml'] + if 'filebeat_version' in $facts and $facts['filebeat_version'] != false { + $skip_validation = versioncmp($facts['filebeat_version'], $filebeat::major_version) ? { + -1 => true, + default => false, + } + } else { + $skip_validation = false + } - case $::kernel { + case $facts['kernel'] { 'Linux' : { - $validate_cmd = $filebeat::disable_config_test ? { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { true => undef, default => $major_version ? { - '5' => "${filebeat::filebeat_path} -N -configtest -c %", - default => "${filebeat::filebeat_path} -c % test config", + '5' => "${filebeat::filebeat_path} ${filebeat::extra_validate_options} -N -configtest -c %", + default => "${filebeat::filebeat_path} ${filebeat::extra_validate_options} -c % test config", }, } - file {'filebeat.yml': + file { 'filebeat.yml': ensure => $filebeat::file_ensure, path => $filebeat::config_file, content => template($filebeat::conf_template), @@ -74,7 +132,7 @@ require => File['filebeat-config-dir'], } - file {'filebeat-config-dir': + file { 'filebeat-config-dir': ensure => $filebeat::directory_ensure, path => $filebeat::config_dir, owner => $filebeat::config_dir_owner, @@ -83,16 +141,75 @@ recurse => $filebeat::purge_conf_dir, purge => $filebeat::purge_conf_dir, force => true, + notify => Service['filebeat'], + } + + file { 'filebeat-modules-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::modules_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], } } # end Linux + 'SunOS' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => '/opt/local/bin/filebeat test config -c %', + } + + file { 'filebeat.yml': + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + owner => $filebeat::config_file_owner, + group => $filebeat::config_file_group, + mode => $filebeat::config_file_mode, + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + + file { 'filebeat-config-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::config_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + } + + file { 'filebeat-modules-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::modules_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + require => File['filebeat-config-dir'], + } + } # end SunOS + 'FreeBSD' : { - $validate_cmd = $filebeat::disable_config_test ? { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { true => undef, - default => '/usr/local/sbin/filebeat -N -configtest -c %', + default => $major_version ? { + '5' => "/usr/local/sbin/filebeat ${filebeat::extra_validate_options} -N -configtest -c %", + default => "/usr/local/sbin/filebeat ${filebeat::extra_validate_options} -c % test config", + }, } - file {'filebeat.yml': + file { 'filebeat.yml': ensure => $filebeat::file_ensure, path => $filebeat::config_file, content => template($filebeat::conf_template), @@ -104,7 +221,7 @@ require => File['filebeat-config-dir'], } - file {'filebeat-config-dir': + file { 'filebeat-config-dir': ensure => $filebeat::directory_ensure, path => $filebeat::config_dir, owner => $filebeat::config_dir_owner, @@ -113,19 +230,83 @@ recurse => $filebeat::purge_conf_dir, purge => $filebeat::purge_conf_dir, force => true, + notify => Service['filebeat'], + } + + file { 'filebeat-modules-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::modules_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], } } # end FreeBSD + 'OpenBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $major_version ? { + '5' => "${filebeat::filebeat_path} ${filebeat::extra_validate_options} -N -configtest -c %", + default => "${filebeat::filebeat_path} ${filebeat::extra_validate_options} -c % test config", + }, + } + + file { 'filebeat.yml': + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + owner => $filebeat::config_file_owner, + group => $filebeat::config_file_group, + mode => $filebeat::config_file_mode, + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + + file { 'filebeat-config-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::config_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + notify => Service['filebeat'], + } + + file { 'filebeat-modules-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::modules_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + } # end OpenBSD + 'Windows' : { $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') - $validate_cmd = $filebeat::disable_config_test ? { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { true => undef, - default => "\"${filebeat_path}\" -N -configtest -c \"%\"", + default => $major_version ? { + /(7|8)/ => "\"${filebeat_path}\" ${filebeat::extra_validate_options} test config -c \"%\"", + default => "\"${filebeat_path}\" ${filebeat::extra_validate_options} -N -configtest -c \"%\"", + } } - file {'filebeat.yml': + file { 'filebeat.yml': ensure => $filebeat::file_ensure, path => $filebeat::config_file, content => template($filebeat::conf_template), @@ -134,13 +315,22 @@ require => File['filebeat-config-dir'], } - file {'filebeat-config-dir': + file { 'filebeat-config-dir': ensure => $filebeat::directory_ensure, path => $filebeat::config_dir, recurse => $filebeat::purge_conf_dir, purge => $filebeat::purge_conf_dir, force => true, } + + file { 'filebeat-modules-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::modules_dir, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + require => File['filebeat-config-dir'], + } } # end Windows default : { diff --git a/manifests/init.pp b/manifests/init.pp index 76c2f9da..56b88b58 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -13,81 +13,127 @@ # } # # @param package_ensure [String] The ensure parameter for the filebeat package (default: present) +# @param manage_package [Boolean] Whether ot not to manage the installation of the package (default: true) # @param manage_repo [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) +# @param manage_apt [Boolean] Whether or not the apt class should be explicitly called or not (default: true) # @param major_version [Enum] The major version of Filebeat to be installed. # @param service_ensure [String] The ensure parameter on the filebeat service (default: running) # @param service_enable [String] The enable parameter on the filebeat service (default: true) # @param repo_priority [Integer] Repository priority. yum and apt supported (default: undef) +# @param service_provider [String] The provider parameter on the filebeat service (default: on RedHat based systems use redhat, otherwise undefined) # @param spool_size [Integer] How large the spool should grow before being flushed to the network (default: 2048) # @param idle_timeout [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) # @param publish_async [Boolean] If set to true filebeat will publish while preparing the next batch of lines to send (defualt: false) -# @param registry_file [String] The registry file used to store positions, absolute or relative to working directory (default .filebeat) -# @param config_dir [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) +# @param config_dir [String] The directory where inputs should be defined (default: /etc/filebeat/conf.d) # @param config_dir_mode [String] The unix permissions mode set on the configuration directory (default: 0755) +# @param config_dir_owner [String] The owner of the configuration directory (default: root). Linux only. +# @param config_dir_group [String] The group of the configuration directory (default: root). Linux only. +# @param config_file [String] Where the configuration file managed by this module should be placed. If you think +# you might want to use this, read the [limitations](#using-config_file) first. Defaults to the location +# that filebeat expects for your operating system. # @param config_file_mode [String] The unix permissions mode set on configuration files (default: 0644) -# @param purge_conf_dir [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged +# @param config_file_owner [String] The owner of the configuration files, including inputs (default: root). Linux only. +# @param config_file_group [String] The group of the configuration files, including inputs (default: root). Linux only. +# @param purge_conf_dir [Boolean] Should files in the input configuration directory not managed by puppet be automatically purged +# @param enable_conf_modules [Boolean] Should filebeat.config.modules be enabled +# @param modules_dir [String] The directory where module configurations should be defined (default: /etc/filebeat/modules.d) +# @param http [Hash] A hash of the http section of configuration +# @param cloud [Hash] Will be converted to YAML for the optional cloud of the configuration (see documentation, and above) +# @param features [Hash] Will be converted to YAML to create the optional features section of the filebeat config (see documentation) +# @param queue [Hash] Will be converted to YAML for the optional queue of the configuration (see documentation, and above) # @param outputs [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) # @param shipper [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) # @param logging [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) +# @param modules [Array] Will be converted to YAML to create the optional modules section of the filebeat config (see documentation) # @param conf_template [String] The configuration template to use to generate the main filebeat.yml config file # @param download_url [String] The URL of the zip file that should be downloaded to install filebeat (windows only) # @param install_dir [String] Where filebeat should be installed (windows only) # @param tmp_dir [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only) # @param shutdown_timeout [String] How long filebeat waits on shutdown for the publisher to finish sending events -# @param beat_name [String] The name of the beat shipper (default: hostname) +# @param beat_name [String] The name of the beat shipper (default: FQDN) # @param tags [Array] A list of tags that will be included with each published transaction -# @param queue_size [String] The internal queue size for events in the pipeline # @param max_procs [Integer] The maximum number of CPUs that can be simultaneously used # @param fields [Hash] Optional fields that should be added to each event output # @param fields_under_root [Boolean] If set to true, custom fields are stored in the top level instead of under fields -# @param processors [Hash] Processors that will be added. Commonly used to create processors using hiera. -# @param prospectors [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera -# @param prospectors_merge [Boolean] Whether $prospectors should merge all hiera sources, or use simple automatic parameter lookup +# @param disable_config_test [Boolean] If set to true, configuration tests won't be run on config files before writing them. +# @param ssl [Hash] Optional fields set the ssl-configuration for input +# @param processors [Array] Processors that will be added. Commonly used to create processors using hiera. +# @param monitoring [Hash] The monitoring section of the configuration file. +# @param inputs [Hash] or [Array] Inputs that will be created. Commonly used to create inputs using hiera +# @param setup [Hash] setup that will be created. Commonly used to create setup using hiera # proxy_address [String] Proxy server to use for downloading files +# @param xpack [Hash] Configuration items to export internal stats to a monitoring Elasticsearch cluster +# @param extra_validate_options [String] Extra command line options to pass to the configuration validation command +# @param autodiscover [Hash] Will be converted to YAML for the optional autodiscover section of the configuration (see documentation, and above) +# @param overwrite_pipelines [Boolean] If set to true filebeat will overwrite (ingest) pipeline in Elasticsearch class filebeat ( - String $package_ensure = $filebeat::params::package_ensure, - Boolean $manage_repo = $filebeat::params::manage_repo, - Enum['5','6'] $major_version = $filebeat::params::major_version, - Variant[Boolean, Enum['stopped', 'running']] $service_ensure = $filebeat::params::service_ensure, - Boolean $service_enable = $filebeat::params::service_enable, - Optional[String] $service_provider = $filebeat::params::service_provider, - Optional[Integer] $repo_priority = undef, - Integer $spool_size = $filebeat::params::spool_size, - String $idle_timeout = $filebeat::params::idle_timeout, - Boolean $publish_async = $filebeat::params::publish_async, - String $registry_file = $filebeat::params::registry_file, - String $config_file = $filebeat::params::config_file, - Optional[String] $config_file_owner = $filebeat::params::config_file_owner, - Optional[String] $config_file_group = $filebeat::params::config_file_group, - String[4,4] $config_dir_mode = $filebeat::params::config_dir_mode, - String $config_dir = $filebeat::params::config_dir, - String[4,4] $config_file_mode = $filebeat::params::config_file_mode, - Optional[String] $config_dir_owner = $filebeat::params::config_dir_owner, - Optional[String] $config_dir_group = $filebeat::params::config_dir_group, - Boolean $purge_conf_dir = $filebeat::params::purge_conf_dir, - Hash $outputs = $filebeat::params::outputs, - Hash $shipper = $filebeat::params::shipper, - Hash $logging = $filebeat::params::logging, - Hash $run_options = $filebeat::params::run_options, - String $conf_template = $filebeat::params::conf_template, - Optional[Pattern[/^(http(?:s)?\:\/\/[a-zA-Z0-9]+(?:(?:\.|\-)[a-zA-Z0-9]+)+(?:\:\d+)?(?:\/[\w\-\.]+)*(?:\/?|\/\w+\.[a-zA-Z]{2,4}(?:\?[\w]+\ = [\w\-]+)?)?(?:\&[\w]+\=[\w\-]+)*)$/]] $download_url = undef, # lint:ignore:140chars - Optional[String] $install_dir = $filebeat::params::install_dir, - String $tmp_dir = $filebeat::params::tmp_dir, - Integer $shutdown_timeout = $filebeat::params::shutdown_timeout, - String $beat_name = $filebeat::params::beat_name, - Array $tags = $filebeat::params::tags, - Integer $queue_size = $filebeat::params::queue_size, - Optional[Integer] $max_procs = $filebeat::params::max_procs, - Hash $fields = $filebeat::params::fields, - Boolean $fields_under_root = $filebeat::params::fields_under_root, - Boolean $disable_config_test = $filebeat::params::disable_config_test, - Hash $processors = {}, - Hash $prospectors = {}, - Optional[Pattern[/^(http(?:s)?\:\/\/[a-zA-Z0-9]+(?:(?:\.|\-)[a-zA-Z0-9]+)+(?:\:\d+)?(?:\/[\w\-\.]+)*(?:\/?|\/\w+\.[a-zA-Z]{2,4}(?:\?[\w]+\ = [\w\-]+)?)?(?:\&[\w]+\=[\w\-]+)*)$/]] $proxy_address = undef, # lint:ignore:140chars - Stdlib::Absolutepath $filebeat_path = $filebeat::params::filebeat_path -) inherits filebeat::params { + String $package_ensure = $filebeat::params::package_ensure, + Boolean $manage_package = $filebeat::params::manage_package, + Boolean $manage_repo = $filebeat::params::manage_repo, + Boolean $manage_apt = $filebeat::params::manage_apt, + Enum['5','6', '7', '8'] $major_version = $filebeat::params::major_version, + Variant[Boolean, Enum['stopped', 'running']] $service_ensure = $filebeat::params::service_ensure, + Boolean $service_enable = $filebeat::params::service_enable, + Optional[String] $service_provider = $filebeat::params::service_provider, + Optional[Integer] $repo_priority = undef, + Integer $spool_size = $filebeat::params::spool_size, + String $idle_timeout = $filebeat::params::idle_timeout, + Boolean $publish_async = $filebeat::params::publish_async, + String $config_file = $filebeat::params::config_file, + Optional[String] $config_file_owner = $filebeat::params::config_file_owner, + Optional[String] $config_file_group = $filebeat::params::config_file_group, + String[4,4] $config_dir_mode = $filebeat::params::config_dir_mode, + String $config_dir = $filebeat::params::config_dir, + String[4,4] $config_file_mode = $filebeat::params::config_file_mode, + Optional[String] $config_dir_owner = $filebeat::params::config_dir_owner, + Optional[String] $config_dir_group = $filebeat::params::config_dir_group, + Boolean $purge_conf_dir = $filebeat::params::purge_conf_dir, + String $modules_dir = $filebeat::params::modules_dir, + Boolean $enable_conf_modules = $filebeat::params::enable_conf_modules, + Hash $http = $filebeat::params::http, + Hash $cloud = $filebeat::params::cloud, + Hash $features = $filebeat::params::features, + Hash $queue = $filebeat::params::queue, + Hash $outputs = $filebeat::params::outputs, + Hash $shipper = $filebeat::params::shipper, + Hash $logging = $filebeat::params::logging, + Hash $run_options = $filebeat::params::run_options, + String $conf_template = $filebeat::params::conf_template, + Optional[Variant[Stdlib::HTTPUrl, Stdlib::HTTPSUrl]] $download_url = undef, # lint:ignore:140chars + Optional[String] $install_dir = $filebeat::params::install_dir, + String $tmp_dir = $filebeat::params::tmp_dir, + String $shutdown_timeout = $filebeat::params::shutdown_timeout, + String $beat_name = $filebeat::params::beat_name, + Array $tags = $filebeat::params::tags, + Optional[Integer] $max_procs = $filebeat::params::max_procs, + Hash $fields = $filebeat::params::fields, + Boolean $fields_under_root = $filebeat::params::fields_under_root, + Hash $ssl = $filebeat::params::ssl, + Boolean $disable_config_test = $filebeat::params::disable_config_test, + Array $processors = [], + Optional[Hash] $monitoring = undef, + Variant[Hash, Array] $inputs = {}, + Hash $setup = {}, + Array $modules = [], + Optional[Variant[Stdlib::HTTPUrl, Stdlib::HTTPSUrl]] $proxy_address = undef, # lint:ignore:140chars + Stdlib::Absolutepath $filebeat_path = $filebeat::params::filebeat_path, + Optional[Hash] $xpack = $filebeat::params::xpack, + + Integer $queue_size = 4096, + String $registry_file = 'filebeat.yml', - include ::stdlib + Optional[String] $systemd_beat_log_opts_override = undef, + String $systemd_beat_log_opts_template = $filebeat::params::systemd_beat_log_opts_template, + String $systemd_override_dir = $filebeat::params::systemd_override_dir, + Optional[String] $extra_validate_options = undef, + Hash $autodiscover = $filebeat::params::autodiscover, + Optional[String] $registry_path = $filebeat::params::registry_path, + Optional[String] $registry_file_permissions = $filebeat::params::registry_file_permissions, + Optional[String] $registry_flush = $filebeat::params::registry_flush, + Boolean $overwrite_pipelines = $filebeat::params::overwrite_pipelines, + +) inherits filebeat::params { + include stdlib $real_download_url = $download_url ? { undef => "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${package_ensure}-windows-${filebeat::params::url_arch}.zip", @@ -103,32 +149,45 @@ $real_service_ensure = 'stopped' $file_ensure = 'absent' $directory_ensure = 'absent' + $real_service_enable = false } else { $alternate_ensure = 'present' $file_ensure = 'file' $directory_ensure = 'directory' $real_service_ensure = $service_ensure + $real_service_enable = $service_enable } # If we're removing filebeat, do things in a different order to make sure # we remove as much as possible if $package_ensure == 'absent' { anchor { 'filebeat::begin': } - -> class { '::filebeat::config': } - -> class { '::filebeat::install': } - -> class { '::filebeat::service': } + -> class { 'filebeat::config': } + -> class { 'filebeat::install': } + -> class { 'filebeat::service': } -> anchor { 'filebeat::end': } } else { - anchor { 'filebeat::begin': } - -> class { '::filebeat::install': } - -> class { '::filebeat::config': } - -> class { '::filebeat::service': } - -> anchor { 'filebeat::end': } + if !$manage_package { + anchor { 'filebeat::begin': } + -> class { 'filebeat::config': } + -> class { 'filebeat::service': } + -> anchor { 'filebeat::end': } + } else { + anchor { 'filebeat::begin': } + -> class { 'filebeat::install': } + -> class { 'filebeat::config': } + -> class { 'filebeat::service': } + -> anchor { 'filebeat::end': } + } } if $package_ensure != 'absent' { - if !empty($prospectors) { - create_resources('filebeat::prospector', $prospectors) + if !empty($inputs) { + if $inputs =~ Array { + create_resources('filebeat::input', { 'inputs' => { pure_array => true } }) + } else { + create_resources('filebeat::input', $inputs) + } } } } diff --git a/manifests/input.pp b/manifests/input.pp new file mode 100644 index 00000000..f56a7027 --- /dev/null +++ b/manifests/input.pp @@ -0,0 +1,165 @@ +# filebeat::input +# +# A description of what this defined type does +# +# @summary A short summary of the purpose of this defined type. +# +# @example +# filebeat::input { 'namevar': } +define filebeat::input ( + Enum['absent', 'present'] $ensure = present, + Array[String] $paths = [], + Array[String] $exclude_files = [], + Array[String] $containers_ids = ['\'*\''], + String $containers_path = '/var/lib/docker/containers', + String $containers_stream = 'all', + Boolean $combine_partial = false, + Enum['tcp', 'udp'] $syslog_protocol = 'udp', + String $syslog_host = 'localhost:5140', + Boolean $cri_parse_flags = false, + String $encoding = 'plain', + String $input_type = $filebeat::params::default_input_type, + Optional[Boolean] $take_over = undef, + Hash $fields = {}, + Boolean $fields_under_root = $filebeat::fields_under_root, + Hash $ssl = {}, + Optional[String] $ignore_older = undef, + Optional[String] $close_older = undef, + String $doc_type = 'log', + String $scan_frequency = '10s', + Integer $harvester_buffer_size = 16384, + Optional[Integer] $harvester_limit = undef, + Boolean $tail_files = false, + String $backoff = '1s', + String $max_backoff = '10s', + Integer $backoff_factor = 2, + String $close_inactive = '5m', + Boolean $close_renamed = false, + Boolean $close_removed = true, + Boolean $close_eof = false, + Variant[String, Integer] $clean_inactive = 0, + Boolean $clean_removed = true, + Variant[Integer,String] $close_timeout = 0, + Boolean $force_close_files = false, + Array[String] $include_lines = [], + Array[String] $exclude_lines = [], + String $max_bytes = '10485760', + Hash $multiline = {}, + Hash $json = {}, + Array[String] $tags = [], + Boolean $symlinks = false, + Optional[String] $pipeline = undef, + Array $processors = [], + Boolean $pure_array = false, + String $host = 'localhost:9000', + Boolean $keep_null = false, + Array[String] $include_matches = [], + Optional[Enum['head', 'tail', 'cursor']] $seek = undef, + Optional[String] $max_message_size = undef, + Optional[String] $index = undef, + Boolean $publisher_pipeline_disable_host = false, +) { + if 'filebeat_version' in $facts and $facts['filebeat_version'] != false { + if versioncmp($facts['filebeat_version'], '6') > 0 { + $input_template = 'input.yml.erb' + } else { + $input_template = 'prospector.yml.erb' + } + + $skip_validation = versioncmp($facts['filebeat_version'], $filebeat::major_version) ? { + -1 => true, + default => false, + } + } else { + $input_template = 'input.yml.erb' + $skip_validation = false + } + + case $facts['kernel'] { + 'Linux', 'OpenBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $filebeat::major_version ? { + '5' => "\"${filebeat::filebeat_path}\" -N -configtest -c \"%\"", + default => "\"${filebeat::filebeat_path}\" -c \"${filebeat::config_file}\" test config", + }, + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => '0', + mode => $filebeat::config_file_mode, + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + 'SunOS' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => "\"${filebeat::filebeat_path}\" -c \"${filebeat::config_file}\" test config", + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => 'root', + mode => $filebeat::config_file_mode, + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + 'FreeBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $filebeat::major_version ? { + '5' => '/usr/local/sbin/filebeat -N -configtest -c %', + default => "/usr/local/sbin/filebeat -c ${filebeat::config_file} test config", + }, + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => 'wheel', + mode => $filebeat::config_file_mode, + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + 'Windows' : { + $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') + $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') + + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $facts['filebeat_version'] ? { + '5' => "\"${filebeat_path}\" -N -configtest -c \"%\"", + default => "\"${filebeat_path}\" -c \"${filebeat::config_file}\" test config", + }, + } + + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + default : { + fail($filebeat::kernel_fail_message) + } + } +} diff --git a/manifests/install.pp b/manifests/install.pp index 72e21125..0ce2c9bb 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -6,25 +6,35 @@ class filebeat::install { anchor { 'filebeat::install::begin': } - case $::kernel { + case $facts['kernel'] { 'Linux': { - class{ '::filebeat::install::linux': + class { 'filebeat::install::linux': notify => Class['filebeat::service'], } Anchor['filebeat::install::begin'] -> Class['filebeat::install::linux'] -> Anchor['filebeat::install::end'] - if $::filebeat::manage_repo { - class { '::filebeat::repo': } + if $filebeat::manage_repo { + class { 'filebeat::repo': } Class['filebeat::repo'] -> Class['filebeat::install::linux'] } } + 'SunOS': { + class { 'filebeat::install::sunos': + notify => Class['filebeat::service'], + } + Anchor['filebeat::install::begin'] -> Class['filebeat::install::sunos'] -> Anchor['filebeat::install::end'] + } 'FreeBSD': { - class{ '::filebeat::install::freebsd': + class { 'filebeat::install::freebsd': notify => Class['filebeat::service'], } Anchor['filebeat::install::begin'] -> Class['filebeat::install::freebsd'] -> Anchor['filebeat::install::end'] } + 'OpenBSD': { + class { 'filebeat::install::openbsd': } + Anchor['filebeat::install::begin'] -> Class['filebeat::install::openbsd'] -> Anchor['filebeat::install::end'] + } 'Windows': { - class{'::filebeat::install::windows': + class { 'filebeat::install::windows': notify => Class['filebeat::service'], } Anchor['filebeat::install::begin'] -> Class['filebeat::install::windows'] -> Anchor['filebeat::install::end'] @@ -35,5 +45,4 @@ } anchor { 'filebeat::install::end': } - } diff --git a/manifests/install/freebsd.pp b/manifests/install/freebsd.pp index e67ee8ef..c02d9df7 100644 --- a/manifests/install/freebsd.pp +++ b/manifests/install/freebsd.pp @@ -5,9 +5,7 @@ # @summary A simple class to install the filebeat package # class filebeat::install::freebsd { - # filebeat, heartbeat, metricbeat, packetbeat are all contained in a # single FreeBSD Package (see https://www.freshports.org/sysutils/beats/ ) - ensure_packages (['beats'], {ensure => $filebeat::package_ensure}) - + ensure_packages (["beats${filebeat::major_version}"], { ensure => $filebeat::package_ensure }) } diff --git a/manifests/install/linux.pp b/manifests/install/linux.pp index bbe4db85..1969ef21 100644 --- a/manifests/install/linux.pp +++ b/manifests/install/linux.pp @@ -5,7 +5,11 @@ # @summary A simple class to install the filebeat package # class filebeat::install::linux { - package {'filebeat': + if $facts['kernel'] != 'Linux' { + fail('filebeat::install::linux shouldn\'t run on Windows') + } + + package { 'filebeat': ensure => $filebeat::package_ensure, } } diff --git a/manifests/install/openbsd.pp b/manifests/install/openbsd.pp new file mode 100644 index 00000000..b15beb7d --- /dev/null +++ b/manifests/install/openbsd.pp @@ -0,0 +1,6 @@ +# to manage filebeat installation on OpenBSD +class filebeat::install::openbsd { + package { 'filebeat': + ensure => $filebeat::package_ensure, + } +} diff --git a/manifests/install/sunos.pp b/manifests/install/sunos.pp new file mode 100644 index 00000000..6a1792f0 --- /dev/null +++ b/manifests/install/sunos.pp @@ -0,0 +1,6 @@ +# to manage filebeat installation on SunOS +class filebeat::install::sunos { + package { 'beats': + ensure => $filebeat::package_ensure, + } +} diff --git a/manifests/install/windows.pp b/manifests/install/windows.pp index 7b831455..b2fdd48d 100644 --- a/manifests/install/windows.pp +++ b/manifests/install/windows.pp @@ -37,8 +37,19 @@ proxy_server => $filebeat::proxy_address, } + # Core editions of Windows Server do not have a shell as such, so use the Shell.Application COM object doesn't work. + # Expand-Archive is a native powershell cmdlet which ships with Powershell 5, which in turn ships with Windows 10 and + # Windows Server 2016 and newer. + if ( (versioncmp($facts['os']['release']['full'], '2016') >= 0) + or (versioncmp($facts['os']['release']['full'], '2000') < 0 and versioncmp($facts['os']['release']['full'], '10') >= 0) ) { + $unzip_command = "Expand-Archive ${zip_file} \"${filebeat::install_dir}\"" + } + else { + $unzip_command = "\$sh=New-Object -COM Shell.Application;\$sh.namespace((Convert-Path '${filebeat::install_dir}')).Copyhere(\$sh.namespace((Convert-Path '${zip_file}')).items(), 16)" # lint:ignore:140chars + } + exec { "unzip ${filename}": - command => "\$sh=New-Object -COM Shell.Application;\$sh.namespace((Convert-Path '${filebeat::install_dir}')).Copyhere(\$sh.namespace((Convert-Path '${zip_file}')).items(), 16)", # lint:ignore:140chars + command => $unzip_command, creates => $version_file, require => [ File[$filebeat::install_dir], diff --git a/manifests/module.pp b/manifests/module.pp new file mode 100644 index 00000000..4e282c57 --- /dev/null +++ b/manifests/module.pp @@ -0,0 +1,65 @@ +# filebeat::module +# +# @summary Base resource to manage Filebeat modules. Check filebeat::module::* for specific implementations. +# +# @example +# filebeat::module { 'namevar': +# config => { +# 'log' => { +# 'enabled' => true, +# 'var.paths' => [ '/var/log/*.log' ], +# }, +# }, +# } +# +# @param ensure Present or absent. Default: present. +# @param config Hash with the module configuration. +# +define filebeat::module ( + Enum['absent', 'present'] $ensure = present, + Hash $config = {}, +) { + $filebeat_config = [{ 'module' => $name } + $config] + + case $facts['kernel'] { + 'Linux', 'OpenBSD' : { + file { "filebeat-module-${name}": + ensure => $ensure, + path => "${filebeat::modules_dir}/${name}.yml", + owner => 'root', + group => '0', + mode => $filebeat::config_file_mode, + content => template("${module_name}/pure_hash.yml.erb"), + notify => Service['filebeat'], + before => File['filebeat.yml'], + } + } + + 'FreeBSD' : { + file { "filebeat-module-${name}": + ensure => $ensure, + path => "${filebeat::modules_dir}/${name}.yml", + owner => 'root', + group => 'wheel', + mode => $filebeat::config_file_mode, + content => template("${module_name}/pure_hash.yml.erb"), + notify => Service['filebeat'], + before => File['filebeat.yml'], + } + } + + 'Windows' : { + file { "filebeat-module-${name}": + ensure => $ensure, + path => "${filebeat::modules_dir}/${name}.yml", + content => template("${module_name}/pure_hash.yml.erb"), + notify => Service['filebeat'], + before => File['filebeat.yml'], + } + } + + default : { + fail($filebeat::kernel_fail_message) + } + } +} diff --git a/manifests/module/apache.pp b/manifests/module/apache.pp new file mode 100644 index 00000000..19d979ba --- /dev/null +++ b/manifests/module/apache.pp @@ -0,0 +1,49 @@ +# filebeat::module::apache +# +# @summary +# This class manages the Filebeat module for Apache HTTP Server. +# +# @example +# class { 'filebeat::module::apache': +# access_enabled => true, +# access_paths => [ +# '/var/log/apache2/access.log', +# ], +# error_enabled => true, +# error_paths => [ +# '/var/log/apache2/error.log', +# ], +# } +# +# @param access_enabled +# Whether to enable the Apache access log module. Defaults to `false`. +# @param access_paths +# An array of absolute paths to Apache access log files. Defaults to `undef`. +# @param error_enabled +# Whether to enable the Apache error log module. Defaults to `false`. +# @param error_paths +# An array of absolute paths to Apache error log files. Defaults to `undef`. +# +class filebeat::module::apache ( + Boolean $access_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $access_paths = undef, + Boolean $error_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $error_paths = undef, +) { + filebeat::module { 'apache': + config => { + 'access' => delete_undef_values( + { + 'enabled' => $access_enabled, + 'var.paths' => $access_paths, + } + ), + 'error' => delete_undef_values( + { + 'enabled' => $error_enabled, + 'var.paths' => $error_paths, + } + ), + }, + } +} diff --git a/manifests/module/auditd.pp b/manifests/module/auditd.pp new file mode 100644 index 00000000..f4894fef --- /dev/null +++ b/manifests/module/auditd.pp @@ -0,0 +1,33 @@ +# filebeat::module::auditd +# +# @summary +# This class manages the Filebeat module for auditd. +# +# @example +# class { 'filebeat::module::auditd': +# log_enabled => true, +# log_paths => [ +# '/var/log/audit/audit.log', +# ], +# } +# +# @param log_enabled +# Whether to enable the auditd module. +# @param log_paths +# An array of absolute paths to the auditd log files. +# +class filebeat::module::auditd ( + Boolean $log_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, +) { + filebeat::module { 'auditd': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.paths' => $log_paths, + } + ), + }, + } +} diff --git a/manifests/module/elasticsearch.pp b/manifests/module/elasticsearch.pp new file mode 100644 index 00000000..ad238528 --- /dev/null +++ b/manifests/module/elasticsearch.pp @@ -0,0 +1,87 @@ +# filebeat::module::elasticsearch +# +# @summary +# This class manages the filebeat module for elasticsearch. +# +# @example +# class { 'filebeat::module::elasticsearch': +# server_enabled => true, +# server_paths => ['/var/log/elasticsearch/*.log'], +# gc_enabled => true, +# gc_paths => ['/var/log/elasticsearch/gc.log*'], +# audit_enabled => true, +# audit_paths => ['/var/log/elasticsearch/audit.log*'], +# deprecation_enabled => true, +# deprecation_paths => ['/var/log/elasticsearch/deprecation.log*'], +# slowlog_enabled => true, +# slowlog_paths => ['/var/log/elasticsearch/*_index_search_slowlog.log'], +# } +# +# @param server_enabled +# Boolean to enable or disable the server log. Defaults to false. +# @param server_paths +# Array of absolute paths to the server log files. Defaults to undef. +# @param gc_enabled +# Boolean to enable or disable the garbage collection log. Defaults to false. +# @param gc_paths +# Array of absolute paths to the garbage collection log files. Defaults to undef. +# @param audit_enabled +# Boolean to enable or disable the audit log. Defaults to false. +# @param audit_paths +# Array of absolute paths to the audit log files. Defaults to undef. +# @param deprecation_enabled +# Boolean to enable or disable the deprecation log. Defaults to false. +# @param deprecation_paths +# Array of absolute paths to the deprecation log files. Defaults to undef. +# @param slowlog_enabled +# Boolean to enable or disable the slow log. Defaults to false. +# @param slowlog_paths +# Array of absolute paths to the slow log files. Defaults to undef. +# +class filebeat::module::elasticsearch ( + Boolean $server_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $server_paths = undef, + Boolean $gc_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $gc_paths = undef, + Boolean $audit_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $audit_paths = undef, + Boolean $deprecation_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $deprecation_paths = undef, + Boolean $slowlog_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $slowlog_paths = undef, +) { + filebeat::module { 'elasticsearch': + config => { + 'server' => delete_undef_values( + { + 'enabled' => $server_enabled, + 'var.paths' => $server_paths, + } + ), + 'gc' => delete_undef_values( + { + 'enabled' => $gc_enabled, + 'var.paths' => $gc_paths, + } + ), + 'audit' => delete_undef_values( + { + 'enabled' => $audit_enabled, + 'var.paths' => $audit_paths, + } + ), + 'deprecation' => delete_undef_values( + { + 'enabled' => $deprecation_enabled, + 'var.paths' => $deprecation_paths, + } + ), + 'slowlog' => delete_undef_values( + { + 'enabled' => $slowlog_enabled, + 'var.paths' => $slowlog_paths, + } + ), + }, + } +} diff --git a/manifests/module/iptables.pp b/manifests/module/iptables.pp new file mode 100644 index 00000000..2cc46c1f --- /dev/null +++ b/manifests/module/iptables.pp @@ -0,0 +1,55 @@ +# filebeat::module::iptables +# +# @summary +# This class manages the Filebeat iptables module. +# +# @example +# class { 'filebeat::module::iptables': +# log_enabled => true, +# log_paths => [ +# '/var/log/iptables.log', +# ], +# log_syslog_host => '0.0.0.0', +# log_syslog_port => 9001, +# log_tags => [ +# 'iptables', +# 'forwarded', +# ], +# } +# +# @param log_enabled +# Whether to enable the iptables module. Defaults to `false`. +# @param log_input +# The input to use for the iptables logs. file to read from a file, syslog to listen for syslog messages. +# @param log_paths +# An array of absolute paths to the iptables log files. +# @param log_syslog_host +# The interface to bind to for listening for syslog messages. +# @param log_syslog_port +# The port to listen on for syslog messages. +# @param log_tags +# An array of tags to add to the iptables logs. +# +class filebeat::module::iptables ( + Boolean $log_enabled = false, + Optional[Enum['file', 'syslog']] $log_input = undef, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, + Optional[Stdlib::Host] $log_syslog_host = undef, + Optional[Stdlib::Port] $log_syslog_port = undef, + Optional[Array[String[1]]] $log_tags = undef, +) { + filebeat::module { 'iptables': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.input' => $log_input, + 'var.paths' => $log_paths, + 'var.syslog_host' => $log_syslog_host, + 'var.syslog_port' => $log_syslog_port, + 'var.tags' => $log_tags, + } + ), + }, + } +} diff --git a/manifests/module/kibana.pp b/manifests/module/kibana.pp new file mode 100644 index 00000000..d75183d1 --- /dev/null +++ b/manifests/module/kibana.pp @@ -0,0 +1,45 @@ +# filebeat::module::kibana +# +# @summary +# This class manages the Filebeat Kibana module. +# +# @example +# class { 'filebeat::module::kibana': +# log_enabled => true, +# log_paths => ['/var/log/kibana.log'], +# audit_enabled => true, +# audit_paths => ['/var/log/kibana-audit.log'], +# } +# +# @param log_enabled +# Whether to enable the Kibana log input. Defaults to `false`. +# @param log_paths +# An array of absolute paths to the Kibana log files. +# @param audit_enabled +# Whether to enable the Kibana audit input. Defaults to `false`. +# @param audit_paths +# An array of absolute paths to the Kibana audit log files. +# +class filebeat::module::kibana ( + Boolean $log_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, + Boolean $audit_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $audit_paths = undef, +) { + filebeat::module { 'kibana': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.paths' => $log_paths, + } + ), + 'audit' => delete_undef_values( + { + 'enabled' => $audit_enabled, + 'var.paths' => $audit_paths, + } + ), + }, + } +} diff --git a/manifests/module/logstash.pp b/manifests/module/logstash.pp new file mode 100644 index 00000000..97beec42 --- /dev/null +++ b/manifests/module/logstash.pp @@ -0,0 +1,45 @@ +# filebeat::module::logstash +# +# @summary +# This class manages the Filebeat Logstash module. +# +# @example +# class { 'filebeat::module::logstash': +# log_enabled => true, +# log_paths => ['/var/log/logstash/logstash-plain.log'], +# slowlog_enabled => true, +# slowlog_paths => ['/var/log/logstash/logstash-slowlog.log'], +# } +# +# @param log_enabled +# Whether to enable the Logstash module. +# @param log_paths +# An array of paths to the Logstash logs. +# @param slowlog_enabled +# Whether to enable the Logstash slowlog module. +# @param slowlog_paths +# An array of paths to the Logstash slowlogs. +# +class filebeat::module::logstash ( + Boolean $log_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, + Boolean $slowlog_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $slowlog_paths = undef, +) { + filebeat::module { 'logstash': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.paths' => $log_paths, + } + ), + 'slowlog' => delete_undef_values( + { + 'enabled' => $slowlog_enabled, + 'var.paths' => $slowlog_paths, + } + ), + }, + } +} diff --git a/manifests/module/mysql.pp b/manifests/module/mysql.pp new file mode 100644 index 00000000..f909da26 --- /dev/null +++ b/manifests/module/mysql.pp @@ -0,0 +1,49 @@ +# filebeat::module::mysql +# +# @summary +# This class manages the Filebeat module for MySQL. +# +# @example +# class { 'filebeat::module::mysql': +# error_enabled => true, +# error_paths => [ +# '/var/log/mysql/error.log', +# ], +# slowlog_enabled => true, +# slowlog_paths => [ +# '/var/log/mysql/slow.log', +# ], +# } +# +# @param error_enabled +# Whether to enable the MySQL error log module. Defaults to false. +# @param error_paths +# An array of absolute paths to the MySQL error log files. Defaults to undef. +# @param slowlog_enabled +# Whether to enable the MySQL slow log module. Defaults to false. +# @param slowlog_paths +# An array of absolute paths to the MySQL slow log files. Defaults to undef. +# +class filebeat::module::mysql ( + Boolean $error_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $error_paths = undef, + Boolean $slowlog_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $slowlog_paths = undef, +) { + filebeat::module { 'mysql': + config => { + 'error' => delete_undef_values( + { + 'enabled' => $error_enabled, + 'var.paths' => $error_paths, + } + ), + 'slowlog' => delete_undef_values( + { + 'enabled' => $slowlog_enabled, + 'var.paths' => $slowlog_paths, + } + ), + }, + } +} diff --git a/manifests/module/nginx.pp b/manifests/module/nginx.pp new file mode 100644 index 00000000..f8ce7652 --- /dev/null +++ b/manifests/module/nginx.pp @@ -0,0 +1,65 @@ +# filebeat::module::nginx +# +# @summary +# This class manages the Filebeat module for Nginx. +# +# @example +# class { 'filebeat::module::nginx': +# access_enabled => true, +# access_paths => [ +# '/var/log/nginx/access.log*', +# ], +# error_enabled => true, +# error_paths => [ +# '/var/log/nginx/error.log*', +# ], +# ingress_enabled => true, +# ingress_paths => [ +# '/var/log/nginx/ingress.log*', +# ], +# } +# +# @param access_enabled +# Whether to enable the Nginx access module. +# @param access_paths +# The paths to the Nginx access logs. +# @param error_enabled +# Whether to enable the Nginx error module. +# @param error_paths +# The paths to the Nginx error logs. +# @param ingress_controller_enabled +# Whether to enable the Nginx ingress_controller module. +# @param ingress_controller_paths +# The paths to the Nginx ingress_controller logs. +# +class filebeat::module::nginx ( + Boolean $access_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $access_paths = undef, + Boolean $error_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $error_paths = undef, + Boolean $ingress_controller_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $ingress_controller_paths = undef, +) { + filebeat::module { 'nginx': + config => { + 'access' => delete_undef_values( + { + 'enabled' => $access_enabled, + 'var.paths' => $access_paths, + } + ), + 'error' => delete_undef_values( + { + 'enabled' => $error_enabled, + 'var.paths' => $error_paths, + } + ), + 'ingress_controller' => delete_undef_values( + { + 'enabled' => $ingress_controller_enabled, + 'var.paths' => $ingress_controller_paths, + } + ), + }, + } +} diff --git a/manifests/module/postgresql.pp b/manifests/module/postgresql.pp new file mode 100644 index 00000000..31db1e7f --- /dev/null +++ b/manifests/module/postgresql.pp @@ -0,0 +1,33 @@ +# filebeat::module::postgresql +# +# @summary +# This class manages the Filebeat module for PostgreSQL. +# +# @example +# class { 'filebeat::module::postgresql': +# log_enabled => true, +# log_paths => [ +# '/var/log/postgresql/*.log', +# ], +# } +# +# @param log_enabled +# Whether to enable the PostgreSQL module. +# @param log_paths +# An array of absolute paths to the PostgreSQL log files. +# +class filebeat::module::postgresql ( + Boolean $log_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, +) { + filebeat::module { 'postgresql': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.paths' => $log_paths, + } + ), + }, + } +} diff --git a/manifests/module/rabbitmq.pp b/manifests/module/rabbitmq.pp new file mode 100644 index 00000000..19d0b117 --- /dev/null +++ b/manifests/module/rabbitmq.pp @@ -0,0 +1,31 @@ +# filebeat::module::rabbitmq +# +# @summary +# This class manages the Filebeat RabbitMQ module. +# +# @example +# class { 'filebeat::module::rabbitmq': +# log_enabled => true, +# log_paths => ['/var/log/rabbitmq/*.log'], +# } +# +# @param log_enabled +# Whether to enable the RabbitMQ module. +# @param log_paths +# An array of paths to the RabbitMQ log files. +# +class filebeat::module::rabbitmq ( + Boolean $log_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, +) { + filebeat::module { 'rabbitmq': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.paths' => $log_paths, + } + ), + }, + } +} diff --git a/manifests/module/redis.pp b/manifests/module/redis.pp new file mode 100644 index 00000000..92b82984 --- /dev/null +++ b/manifests/module/redis.pp @@ -0,0 +1,50 @@ +# filebeat::module::redis +# +# @summary +# This class manages the Filebeat Redis module. +# +# @example +# class { 'filebeat::module::redis': +# log_enabled => true, +# log_paths => ['/var/log/redis/redis-server.log'], +# slowlog_enabled => true, +# slowlog_hosts => ['localhost:6379'], +# slowlog_password => 'password', +# } +# +# @param log_enabled +# Whether to enable the Redis log input. Defaults to `false`. +# @param log_paths +# The paths to the Redis log files. Defaults to `undef`. +# @param slowlog_enabled +# Whether to enable the Redis slowlog input. Defaults to `false`. +# @param slowlog_hosts +# The Redis hosts to connect to. Defaults to `undef`. +# @param slowlog_password +# The password to use when connecting to Redis. Defaults to `undef`. +# +class filebeat::module::redis ( + Boolean $log_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $log_paths = undef, + Boolean $slowlog_enabled = false, + Optional[Array[String[1]]] $slowlog_hosts = undef, + Optional[String[1]] $slowlog_password = undef, +) { + filebeat::module { 'redis': + config => { + 'log' => delete_undef_values( + { + 'enabled' => $log_enabled, + 'var.paths' => $log_paths, + } + ), + 'slowlog' => delete_undef_values( + { + 'enabled' => $slowlog_enabled, + 'var.hosts' => $slowlog_hosts, + 'var.password' => $slowlog_password, + } + ), + }, + } +} diff --git a/manifests/module/sophos.pp b/manifests/module/sophos.pp new file mode 100644 index 00000000..835feaf4 --- /dev/null +++ b/manifests/module/sophos.pp @@ -0,0 +1,86 @@ +# filebeat::module::sophos +# +# @summary +# This class manages the Filebeat Sophos module. +# +# @example +# class { 'filebeat::module::sophos': +# xg_enabled => true, +# xg_input => 'udp', +# xg_syslog_host => '0.0.0.0', +# xg_syslog_port => 514, +# xg_host_name => 'sophos-xg', +# } +# +# @param xg_enabled +# Whether to enable the Sophos XG module. +# @param xg_paths +# An array of paths to the Sophos XG logs. +# @param xg_input +# The input type for the Sophos XG module. tcp or udp for syslog input, file for log files. +# @param xg_syslog_host +# Interface to listen to for syslog input. +# @param xg_syslog_port +# Port to listen on for syslog input. +# @param xg_host_name +# Host name / Observer name, since SophosXG does not provide this in the syslog file. +# @param utm_enabled +# Whether to enable the Sophos UTM module. +# @param utm_paths +# An array of paths to the Sophos UTM logs. +# @param utm_input +# The input type for the Sophos UTM module. tcp or udp for syslog input, file for log files. +# @param utm_syslog_host +# Interface to listen to for syslog input. +# @param utm_syslog_port +# Port to listen on for syslog input. +# @param utm_tz_offset +# Timezone offset. If the logs are in a different timezone than the Filebeat host, set this to the timezone offset. +# @param utm_rsa_fields +# Flag to control whether non-ECS fields are added to the event. +# @param utm_keep_raw_fields +# Flag to control the addition of the raw parser fields to the event. +# +class filebeat::module::sophos ( + Boolean $xg_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $xg_paths = undef, + Optional[Enum['udp', 'tcp','file']] $xg_input = undef, + Optional[Stdlib::Host] $xg_syslog_host = undef, + Optional[Stdlib::Port] $xg_syslog_port = undef, + Optional[Stdlib::Host] $xg_host_name = undef, + Boolean $utm_enabled = false, + Optional[Array[Stdlib::Absolutepath]] $utm_paths = undef, + Optional[Enum['udp', 'tcp','file']] $utm_input = undef, + Optional[Stdlib::Host] $utm_syslog_host = undef, + Optional[Stdlib::Port] $utm_syslog_port = undef, + Optional[Pattern[/^[-+]\d{2}:\d{2}$/]] $utm_tz_offset = undef, + Optional[Boolean] $utm_rsa_fields = undef, + Optional[Boolean] $utm_keep_raw_fields = undef, +) { + filebeat::module { 'sophos': + config => { + 'xg' => delete_undef_values( + { + 'enabled' => $xg_enabled, + 'var.input' => $xg_input, + 'var.paths' => $xg_paths, + 'var.syslog_host' => $xg_syslog_host, + 'var.syslog_port' => $xg_syslog_port, + 'var.host_name' => $xg_host_name, + } + ), + 'utm' => delete_undef_values( + { + 'enabled' => $utm_enabled, + 'var.input' => $utm_input, + 'var.paths' => $utm_paths, + 'var.syslog_host' => $utm_syslog_host, + 'var.syslog_port' => $utm_syslog_port, + 'var.tz_offset' => $utm_tz_offset, + 'var.rsa_fields' => $utm_rsa_fields, + 'var.keep_raw_fields' => $utm_keep_raw_fields, + } + ), + }, + } +} diff --git a/manifests/module/system.pp b/manifests/module/system.pp new file mode 100644 index 00000000..a75b9ab7 --- /dev/null +++ b/manifests/module/system.pp @@ -0,0 +1,49 @@ +# filebeat::module::system +# +# @summary +# This class manages the Filebeat system module. +# +# @example +# class { 'filebeat::module::system': +# syslog_enabled => true, +# syslog_paths => [ +# '/var/log/syslog', +# ], +# auth_enabled => true, +# auth_paths => [ +# '/var/log/auth.log', +# ], +# } +# +# @param syslog_enabled +# A boolean value to enable or disable the syslog module. +# @param syslog_paths +# An optional array of paths to the syslog logs. +# @param auth_enabled +# A boolean value to enable or disable the auth module. +# @param auth_paths +# An optional array of paths to the auth logs. +# +class filebeat::module::system ( + Boolean $syslog_enabled = false, + Optional[Array[Stdlib::Unixpath]] $syslog_paths = undef, + Boolean $auth_enabled = false, + Optional[Array[Stdlib::Unixpath]] $auth_paths = undef, +) { + filebeat::module { 'system': + config => { + 'syslog' => delete_undef_values( + { + 'enabled' => $syslog_enabled, + 'var.paths' => $syslog_paths, + } + ), + 'auth' => delete_undef_values( + { + 'enabled' => $auth_enabled, + 'var.paths' => $auth_paths, + } + ), + }, + } +} diff --git a/manifests/params.pp b/manifests/params.pp index a44097e0..ae794879 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -4,28 +4,44 @@ # # @summary Set a bunch of default parameters class filebeat::params { - $service_ensure = running - $service_enable = true - $spool_size = 2048 - $idle_timeout = '5s' - $publish_async = false - $shutdown_timeout = 0 - $beat_name = $::fqdn - $tags = [] - $queue_size = 1000 - $max_procs = undef - $config_file_mode = '0644' - $config_dir_mode = '0755' - $purge_conf_dir = true - $fields = {} - $fields_under_root = false - $outputs = {} - $shipper = {} - $logging = {} - $run_options = {} - $kernel_fail_message = "${::kernel} is not supported by filebeat." - $conf_template = "${module_name}/pure_hash.yml.erb" - $disable_config_test = false + $manage_package = true + $service_ensure = running + $service_enable = true + $spool_size = 2048 + $idle_timeout = '5s' + $publish_async = false + $shutdown_timeout = '0' + $beat_name = $facts['networking']['fqdn'] + $tags = [] + $max_procs = undef + $config_file_mode = '0644' + $config_dir_mode = '0755' + $purge_conf_dir = true + $enable_conf_modules = false + $fields = {} + $fields_under_root = false + $ssl = {} + $http = {} + $cloud = {} + $features = {} + $queue = {} + $outputs = {} + $shipper = {} + $logging = {} + $autodiscover = {} + $run_options = {} + $modules = [] + $overwrite_pipelines = false + $kernel_fail_message = "${facts['kernel']} is not supported by filebeat." + $osfamily_fail_message = "${facts['os']['family']} is not supported by filebeat." + $conf_template = "${module_name}/pure_hash.yml.erb" + $disable_config_test = false + $xpack = undef + $systemd_override_dir = '/etc/systemd/system/filebeat.service.d' + $systemd_beat_log_opts_template = "${module_name}/systemd/logging.conf.erb" + $registry_path = '/var/lib/filebeat' + $registry_file_permissions = '0600' + $registry_flush = '0s' # These are irrelevant as long as the template is set based on the major_version parameter # if versioncmp('1.9.1', $::rubyversion) > 0 { @@ -35,21 +51,34 @@ # } # - # Archlinux has a proper package in the official repos - # we shouldn't manage the repo on it + # Archlinux and OpenBSD have proper packages in the official repos + # we shouldn't manage the repo on them case $facts['os']['family'] { 'Archlinux': { $manage_repo = false + $manage_apt = false $filebeat_path = '/usr/bin/filebeat' - $major_version = '6' + $major_version = '7' + } + 'OpenBSD': { + $manage_repo = false + $manage_apt = false + $filebeat_path = '/usr/local/bin/filebeat' + # lint:ignore:only_variable_string + $major_version = versioncmp('6.3', $facts['kernelversion']) < 0 ? { + # lint:endignore + true => '6', + default => '5' + } } default: { $manage_repo = true + $manage_apt = true $filebeat_path = '/usr/share/filebeat/bin/filebeat' - $major_version = '5' + $major_version = '8' } } - case $::kernel { + case $facts['kernel'] { 'Linux' : { $package_ensure = present $config_file = '/etc/filebeat/filebeat.yml' @@ -58,14 +87,13 @@ $config_file_group = 'root' $config_dir_owner = 'root' $config_dir_group = 'root' - $registry_file = '/var/lib/filebeat/registry' - + $modules_dir = '/etc/filebeat/modules.d' # These parameters are ignored if/until tarball installs are supported in Linux $tmp_dir = '/tmp' $install_dir = undef - case $::osfamily { + case $facts['os']['family'] { 'RedHat': { - $service_provider = 'redhat' + $service_provider = 'systemd' } default: { $service_provider = undef @@ -74,15 +102,45 @@ $url_arch = undef } + 'SunOS': { + $package_ensure = present + $config_file = '/opt/local/etc/beats/filebeat.yml' + $config_dir = '/opt/local/etc/filebeat.d' + $config_file_owner = 'root' + $config_file_group = 'root' + $config_dir_owner = 'root' + $config_dir_group = 'root' + $modules_dir = '/opt/local/etc/filebeat.modules.d' + $tmp_dir = '/tmp' + $service_provider = undef + $install_dir = undef + $url_arch = undef + } + 'FreeBSD': { $package_ensure = present - $config_file = '/usr/local/etc/filebeat.yml' - $config_dir = '/usr/local/etc/filebeat.d' + $config_file = '/usr/local/etc/beats/filebeat.yml' + $config_dir = '/usr/local/etc/beats/filebeat.d' $config_file_owner = 'root' $config_file_group = 'wheel' $config_dir_owner = 'root' $config_dir_group = 'wheel' - $registry_file = '/var/lib/filebeat/registry' + $modules_dir = '/usr/local/etc/beats/filebeat.modules.d' + $tmp_dir = '/tmp' + $service_provider = undef + $install_dir = undef + $url_arch = undef + } + + 'OpenBSD': { + $package_ensure = present + $config_file = '/etc/filebeat/filebeat.yml' + $config_dir = '/etc/filebeat/conf.d' + $config_file_owner = 'root' + $config_file_group = 'wheel' + $config_dir_owner = 'root' + $config_dir_group = 'wheel' + $modules_dir = '/etc/filebeat/modules.d' $tmp_dir = '/tmp' $service_provider = undef $install_dir = undef @@ -90,21 +148,21 @@ } 'Windows' : { - $package_ensure = '5.6.2' + $package_ensure = '7.1.0' $config_file_owner = 'Administrator' $config_file_group = undef $config_dir_owner = 'Administrator' $config_dir_group = undef $config_file = 'C:/Program Files/Filebeat/filebeat.yml' $config_dir = 'C:/Program Files/Filebeat/conf.d' - $registry_file = 'C:/ProgramData/filebeat/registry' + $modules_dir = 'C:/Program Files/Filebeat/modules.d' $install_dir = 'C:/Program Files' $tmp_dir = 'C:/Windows/Temp' $service_provider = undef - $url_arch = $::architecture ? { + $url_arch = $facts['os']['architecture'] ? { 'x86' => 'x86', 'x64' => 'x86_64', - default => fail("${::architecture} is not supported by filebeat."), + default => fail("${facts['os']['architecture']} is not supported by filebeat."), } } @@ -112,4 +170,15 @@ fail($kernel_fail_message) } } + + if 'filebeat_version' in $facts and $facts['filebeat_version'] != false { + # filestream input type added in 7.10, deprecated in 7.16 + if versioncmp($facts['filebeat_version'], '7.10') > 0 { + $default_input_type = 'filestream' + } else { + $default_input_type = 'log' + } + } else { + $default_input_type = 'filestream' + } } diff --git a/manifests/prospector.pp b/manifests/prospector.pp deleted file mode 100644 index 26ff173d..00000000 --- a/manifests/prospector.pp +++ /dev/null @@ -1,126 +0,0 @@ -# filebeat::prospector -# -# A description of what this defined type does -# -# @summary A short summary of the purpose of this defined type. -# -# @example -# filebeat::prospector { 'namevar': } -define filebeat::prospector ( - $ensure = present, - $paths = [], - $exclude_files = [], - $encoding = 'plain', - $input_type = 'log', - $fields = {}, - $fields_under_root = false, - $ignore_older = undef, - $close_older = undef, - $doc_type = 'log', - $scan_frequency = '10s', - $harvester_buffer_size = 16384, - $tail_files = false, - $backoff = '1s', - $max_backoff = '10s', - $backoff_factor = 2, - $close_inactive = '5m', - $close_renamed = false, - $close_removed = true, - $close_eof = false, - $clean_inactive = 0, - $clean_removed = true, - $close_timeout = 0, - $force_close_files = false, - $include_lines = [], - $exclude_lines = [], - $max_bytes = '10485760', - $multiline = {}, - $json = {}, - $tags = [], - $symlinks = false, - $pipeline = undef, -) { - - validate_hash($fields, $multiline, $json) - validate_array($paths, $exclude_files, $include_lines, $exclude_lines, $tags) - validate_bool($tail_files, $close_renamed, $close_removed, $close_eof, $clean_removed, $symlinks) - - $prospector_template = 'prospector.yml.erb' - - case $::kernel { - 'Linux' : { - if !$filebeat::disable_config_test { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => 'root', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - validate_cmd => "${filebeat::filebeat_path} -N -configtest -c %", - notify => Service['filebeat'], - } - } else { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => 'root', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - notify => Service['filebeat'], - } - } - } - - 'FreeBSD' : { - if !$filebeat::disable_config_test { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => 'wheel', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - validate_cmd => '/usr/local/sbin/filebeat -N -configtest -c %', - notify => Service['filebeat'], - } - } else { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => 'wheel', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - notify => Service['filebeat'], - } - } - } - - 'Windows' : { - $filebeat_path = 'c:\Program Files\Filebeat\filebeat.exe' - if !$filebeat::disable_config_test { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - content => template("${module_name}/${prospector_template}"), - validate_cmd => "\"${filebeat_path}\" -N -configtest -c \"%\"", - notify => Service['filebeat'], - } - } else { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - content => template("${module_name}/${prospector_template}"), - notify => Service['filebeat'], - } - } - } - - default : { - fail($filebeat::kernel_fail_message) - } - - } -} diff --git a/manifests/repo.pp b/manifests/repo.pp index 8133466d..9b87c214 100644 --- a/manifests/repo.pp +++ b/manifests/repo.pp @@ -7,19 +7,21 @@ $debian_repo_url = "https://artifacts.elastic.co/packages/${filebeat::major_version}.x/apt" $yum_repo_url = "https://artifacts.elastic.co/packages/${filebeat::major_version}.x/yum" - case $::osfamily { + case $facts['os']['family'] { 'Debian': { - include ::apt + if $filebeat::manage_apt == true { + include apt + } Class['apt::update'] -> Package['filebeat'] - if !defined(Apt::Source['beats']){ + if !defined(Apt::Source['beats']) { apt::source { 'beats': - ensure => $::filebeat::alternate_ensure, + ensure => $filebeat::alternate_ensure, location => $debian_repo_url, release => 'stable', repos => 'main', - pin => $::filebeat::repo_priority, + pin => $filebeat::repo_priority, key => { id => '46095ACC8548582C1A2699A9D27D666CD88E42B4', source => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', @@ -28,40 +30,46 @@ } } 'RedHat', 'Linux': { - if !defined(Yumrepo['beats']){ + if !defined(Yumrepo['beats']) { yumrepo { 'beats': - ensure => $::filebeat::alternate_ensure, + ensure => $filebeat::alternate_ensure, descr => 'elastic beats repo', baseurl => $yum_repo_url, gpgcheck => 1, gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - priority => $::filebeat::repo_priority, + priority => $filebeat::repo_priority, enabled => 1, + notify => Exec['flush-yum-cache'], } } + + exec { 'flush-yum-cache': + command => 'yum clean all', + refreshonly => true, + path => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'], + } } 'Suse': { exec { 'topbeat_suse_import_gpg': command => 'rpmkeys --import https://artifacts.elastic.co/GPG-KEY-elasticsearch', unless => 'test $(rpm -qa gpg-pubkey | grep -i "D88E42B4" | wc -l) -eq 1 ', - notify => [ Zypprepo['beats'] ], + notify => [Zypprepo['beats']], } - if !defined(Zypprepo['beats']){ + if !defined(Zypprepo['beats']) { zypprepo { 'beats': - ensure => $::filebeat::alternate_ensure, + ensure => $filebeat::alternate_ensure, baseurl => $yum_repo_url, enabled => 1, autorefresh => 1, name => 'beats', gpgcheck => 1, - gpgkey => 'http://packages.elastic.co/GPG-KEY-elasticsearch', + gpgkey => 'https://packages.elastic.co/GPG-KEY-elasticsearch', type => 'yum', } } } default: { - fail($filebeat::kernel_fail_message) + fail($filebeat::osfamily_fail_message) } } - } diff --git a/manifests/service.pp b/manifests/service.pp index 80afd08f..54a623e5 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -6,7 +6,59 @@ class filebeat::service { service { 'filebeat': ensure => $filebeat::real_service_ensure, - enable => $filebeat::service_enable, + enable => $filebeat::real_service_enable, provider => $filebeat::service_provider, } + + $major_version = $filebeat::major_version + $systemd_beat_log_opts_override = $filebeat::systemd_beat_log_opts_override + + #make sure puppet client version 6.1+ with filebeat version 7+, running on systemd + if ( versioncmp( $major_version, '7' ) >= 0 and + $filebeat::service_provider == 'systemd' ) { + if ( versioncmp( $clientversion, '6.1' ) >= 0 ) { + unless $systemd_beat_log_opts_override == undef { + $ensure_overide = 'present' + } else { + $ensure_overide = 'absent' + } + + ensure_resource('file', + $filebeat::systemd_override_dir, + { + ensure => 'directory', + } + ) + + file { "${filebeat::systemd_override_dir}/logging.conf": + ensure => $ensure_overide, + content => template($filebeat::systemd_beat_log_opts_template), + require => File[$filebeat::systemd_override_dir], + notify => Service['filebeat'], + } + } else { + unless $systemd_beat_log_opts_override == undef { + $ensure_overide = 'present' + } else { + $ensure_overide = 'absent' + } + + if !defined(File[$filebeat::systemd_override_dir]) { + file { $filebeat::systemd_override_dir: + ensure => 'directory', + } + } + + file { "${filebeat::systemd_override_dir}/logging.conf": + ensure => $ensure_overide, + content => template($filebeat::systemd_beat_log_opts_template), + require => File[$filebeat::systemd_override_dir], + notify => Service['filebeat'], + } + + unless defined('systemd') { + warning('You\'ve specified an $systemd_beat_log_opts_override varible on a system running puppet version < 6.1 and not declared "systemd" resource See README.md for more information') # lint:ignore:140chars + } + } + } } diff --git a/metadata.json b/metadata.json index c3ad1f28..6ff8f32b 100644 --- a/metadata.json +++ b/metadata.json @@ -1,29 +1,32 @@ { "name": "pcfens-filebeat", - "version": "2.4.0", + "version": "4.14.0", "author": "pcfens", "summary": "A module to install and manage the filebeat log shipper", "license": "Apache-2.0", "source": "https://github.com/pcfens/puppet-filebeat", "project_page": "https://github.com/pcfens/puppet-filebeat", "issues_url": "https://github.com/pcfens/puppet-filebeat/issues", - "tags": ["filebeat", "logstash", "elasticsearch", "elastic"], "dependencies": [ { "name": "puppetlabs/stdlib", - "version_requirement": ">=4.6.0 <5.0.0" + "version_requirement": ">=4.13.0 < 10.0.0" }, { "name": "puppetlabs/apt", - "version_requirement": ">=2.0.0 <5.0.0" + "version_requirement": ">=2.0.0 < 10.0.0" }, { "name": "puppetlabs/powershell", - "version_requirement": ">= 1.0.1 < 3.0.0" + "version_requirement": ">= 1.0.1 < 6.0.0" }, { "name": "puppet/archive", - "version_requirement": ">= 0.5.0 < 3.0.0" + "version_requirement": ">= 0.5.0 < 8.0.0" + }, + { + "name": "puppetlabs/yumrepo_core", + "version_requirement": ">= 1.0.0 < 2.0.0" } ], "operatingsystem_support": [ @@ -39,7 +42,8 @@ "operatingsystem": "OracleLinux", "operatingsystemrelease": [ "6", - "7" + "7", + "8" ] }, { @@ -47,7 +51,8 @@ "operatingsystemrelease": [ "5", "6", - "7" + "7", + "8" ] }, { @@ -64,19 +69,31 @@ "8" ] }, + { + "operatingsystem": "OpenBSD", + "operatingsystemrelease": [ + "6.1", + "6.2", + "6.3" + ] + }, { "operatingsystem": "Ubuntu", "operatingsystemrelease": [ - "12.04", "14.04", - "16.04" + "16.04", + "18.04", + "20.04", + "22.04" ] }, { "operatingsystem": "windows", "operatingsystemrelease": [ "2012", - "2012 R2" + "2012 R2", + "2016", + "2019" ] }, { @@ -86,10 +103,16 @@ "requirements": [ { "name": "puppet", - "version_requirement": ">= 4.0.0 < 6.0.0" + "version_requirement": ">= 4.0.0 < 9.0.0" } ], - "pdk-version": "1.1.0", - "template-url": "file:///opt/puppetlabs/pdk/share/cache/pdk-module-template.git", - "template-ref": "heads/master-0-g54e88a4" + "tags": [ + "filebeat", + "logstash", + "elasticsearch", + "elastic" + ], + "pdk-version": "2.6.1", + "template-url": "pdk-default#2.7.1", + "template-ref": "tags/2.7.1-0-g9a16c87" } diff --git a/pdk.yaml b/pdk.yaml new file mode 100644 index 00000000..4bef4bd0 --- /dev/null +++ b/pdk.yaml @@ -0,0 +1,2 @@ +--- +ignore: [] diff --git a/spec/acceptance/001_basic_spec.rb b/spec/acceptance/001_basic_spec.rb index 8f2bcaa2..9ddb9319 100644 --- a/spec/acceptance/001_basic_spec.rb +++ b/spec/acceptance/001_basic_spec.rb @@ -20,7 +20,7 @@ describe 'filebeat class' do let(:pp) do <<-HEREDOC - if $::osfamily == 'Debian' { + if $facts['os']['family'] == 'Debian' { include ::apt package { 'apt-transport-https': diff --git a/spec/classes/config_spec.rb b/spec/classes/config_spec.rb index 89dcb9cf..93713c95 100644 --- a/spec/classes/config_spec.rb +++ b/spec/classes/config_spec.rb @@ -5,7 +5,7 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}' }" } @@ -24,9 +24,9 @@ case major_version when 5 - "#{path} -N -configtest -c %" + "#{path} -N -configtest -c %" else - "#{path} -c % test config" + "#{path} -c % test config" end end @@ -45,6 +45,13 @@ require: 'File[filebeat-config-dir]', ) } + context 'with added extra_validate_options parameter' do + let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}', extra_validate_options => '--foo'}" } + + it { + is_expected.to contain_file('filebeat.yml').with_validate_cmd(%r{filebeat --foo}) + } + end it { is_expected.to contain_file('filebeat-config-dir').with( diff --git a/spec/classes/filebeat_spec.rb b/spec/classes/filebeat_spec.rb index dfecf6c4..4e860b90 100644 --- a/spec/classes/filebeat_spec.rb +++ b/spec/classes/filebeat_spec.rb @@ -5,7 +5,7 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } diff --git a/spec/classes/install/linux_spec.rb b/spec/classes/install/linux_spec.rb index fbf02ba0..9f3b62fd 100644 --- a/spec/classes/install/linux_spec.rb +++ b/spec/classes/install/linux_spec.rb @@ -5,7 +5,7 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } diff --git a/spec/classes/install/windows_spec.rb b/spec/classes/install/windows_spec.rb index 8dba7eaf..cea15a48 100644 --- a/spec/classes/install/windows_spec.rb +++ b/spec/classes/install/windows_spec.rb @@ -5,52 +5,51 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, facts| context "on #{os}" do - let(:facts) { os_facts } + let(:facts) { facts } - case os_facts[:kernel] + case facts[:kernel] when 'windows' # it { is_expected.to compile } it { is_expected.to contain_file('C:/Program Files').with_ensure('directory') } - it { - is_expected.to contain_archive('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( - creates: 'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64', - ) - } - it { - is_expected.to contain_exec('install filebeat-5.6.2-windows-x86_64').with( - command: './install-service-filebeat.ps1', - ) - } - it { - is_expected.to contain_exec('unzip filebeat-5.6.2-windows-x86_64').with( - command: '$sh=New-Object -COM Shell.Application;$sh.namespace((Convert-Path \'C:/Program Files\')).'\ - 'Copyhere($sh.namespace((Convert-Path \'C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip\')).items(), 16)', - ) - } - it { - is_expected.to contain_exec('mark filebeat-5.6.2-windows-x86_64').with( - command: 'New-Item \'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64\' -ItemType file', - ) - } - it { - is_expected.to contain_exec('rename filebeat-5.6.2-windows-x86_64').with( - command: 'Remove-Item \'C:/Program Files/Filebeat\' -Recurse -Force -ErrorAction SilentlyContinue;'\ - 'Rename-Item \'C:/Program Files/filebeat-5.6.2-windows-x86_64\' \'C:/Program Files/Filebeat\'', - ) - } - it { - is_expected.to contain_exec('stop service filebeat-5.6.2-windows-x86_64').with( - command: 'Set-Service -Name filebeat -Status Stopped', - ) - } - it { - is_expected.to contain_file('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( - ensure: 'absent', - ) - } - + # it { + # is_expected.to contain_archive('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + # creates: 'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64', + # ) + # } + # it { + # is_expected.to contain_exec('install filebeat-5.6.2-windows-x86_64').with( + # command: './install-service-filebeat.ps1', + # ) + # } + # it { + # is_expected.to contain_exec('unzip filebeat-5.6.2-windows-x86_64').with( + # command: '$sh=New-Object -COM Shell.Application;$sh.namespace((Convert-Path \'C:/Program Files\')).'\ + # 'Copyhere($sh.namespace((Convert-Path \'C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip\')).items(), 16)', + # ) + # } + # it { + # is_expected.to contain_exec('mark filebeat-5.6.2-windows-x86_64').with( + # command: 'New-Item \'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64\' -ItemType file', + # ) + # } + # it { + # is_expected.to contain_exec('rename filebeat-5.6.2-windows-x86_64').with( + # command: 'Remove-Item \'C:/Program Files/Filebeat\' -Recurse -Force -ErrorAction SilentlyContinue;'\ + # 'Rename-Item \'C:/Program Files/filebeat-5.6.2-windows-x86_64\' \'C:/Program Files/Filebeat\'', + # ) + # } + # it { + # is_expected.to contain_exec('stop service filebeat-5.6.2-windows-x86_64').with( + # command: 'Set-Service -Name filebeat -Status Stopped', + # ) + # } + # it { + # is_expected.to contain_file('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + # ensure: 'absent', + # ) + # } else it { is_expected.not_to compile } end diff --git a/spec/classes/install_spec.rb b/spec/classes/install_spec.rb index f8b18073..1e69701f 100644 --- a/spec/classes/install_spec.rb +++ b/spec/classes/install_spec.rb @@ -5,7 +5,7 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } diff --git a/spec/classes/module/apache_spec.rb b/spec/classes/module/apache_spec.rb new file mode 100644 index 00000000..53f724ad --- /dev/null +++ b/spec/classes/module/apache_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::apache' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-apache').with_content( + %r{- module: apache\n\s{2}access:\n\s{4}enabled: false\n\s{2}error:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on access and error enabled with paths' do + let(:params) do + { + 'access_enabled' => true, + 'access_paths' => ['/var/log/apache2/access.log', '/var/log/apache2/*-access.log'], + 'error_enabled' => true, + 'error_paths' => ['/var/log/apache2/error.log', '/var/log/apache2/*-error.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-apache').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: apache + access: + enabled: true + var.paths: + - "/var/log/apache2/access.log" + - "/var/log/apache2/*-access.log" + error: + enabled: true + var.paths: + - "/var/log/apache2/error.log" + - "/var/log/apache2/*-error.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/auditd_spec.rb b/spec/classes/module/auditd_spec.rb new file mode 100644 index 00000000..46fdebf4 --- /dev/null +++ b/spec/classes/module/auditd_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::auditd' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-auditd').with_content( + %r{- module: auditd\n\s{2}log:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log enabled with paths' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/audit/audit.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-auditd').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: auditd + log: + enabled: true + var.paths: + - "/var/log/audit/audit.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/elasticsearch_spec.rb b/spec/classes/module/elasticsearch_spec.rb new file mode 100644 index 00000000..72c3feeb --- /dev/null +++ b/spec/classes/module/elasticsearch_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::elasticsearch' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-elasticsearch').with_content( + %r{- module: elasticsearch\n\s{2}server:\n\s{4}enabled: false\n\s{2}gc:\n\s{4}enabled: false\n\s{2}audit:\n\s{4}enabled: false\n\s{2}deprecation:\n\s{4}enabled: false\n\s{2}slowlog:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on server,gc,audit,slowlog and deprecation enabled with paths' do + let(:params) do + { + 'server_enabled' => true, + 'server_paths' => ['/var/log/elasticsearch/*.log'], + 'gc_enabled' => true, + 'gc_paths' => ['/var/log/elasticsearch/gc.log*'], + 'audit_enabled' => true, + 'audit_paths' => ['/var/log/elasticsearch/audit.log'], + 'slowlog_enabled' => true, + 'slowlog_paths' => ['/var/log/elasticsearch/*_search_slowlog.log'], + 'deprecation_enabled' => true, + 'deprecation_paths' => ['/var/log/elasticsearch/*_deprecation.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-elasticsearch').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: elasticsearch + server: + enabled: true + var.paths: + - "/var/log/elasticsearch/*.log" + gc: + enabled: true + var.paths: + - "/var/log/elasticsearch/gc.log*" + audit: + enabled: true + var.paths: + - "/var/log/elasticsearch/audit.log" + deprecation: + enabled: true + var.paths: + - "/var/log/elasticsearch/*_deprecation.log" + slowlog: + enabled: true + var.paths: + - "/var/log/elasticsearch/*_search_slowlog.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/iptables_spec.rb b/spec/classes/module/iptables_spec.rb new file mode 100644 index 00000000..a8602798 --- /dev/null +++ b/spec/classes/module/iptables_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::iptables' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-iptables').with_content( + %r{- module: iptables\n\s{2}log:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on iptables logfile' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/ip6tables.log', '/var/log/iptables.log'], + 'log_input' => 'file', + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-iptables').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: iptables + log: + enabled: true + var.input: file + var.paths: + - "/var/log/ip6tables.log" + - "/var/log/iptables.log" + +EOS + ) + } + end + + context 'on iptables with syslog' do + let(:params) do + { + 'log_enabled' => true, + 'log_input' => 'syslog', + 'log_syslog_host' => '0.0.0.0', + 'log_syslog_port' => 514, + 'log_tags' => [ + "iptables" + ] + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-iptables').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: iptables + log: + enabled: true + var.input: syslog + var.syslog_host: 0.0.0.0 + var.syslog_port: 514 + var.tags: + - iptables + +EOS + ) + } + end +end diff --git a/spec/classes/module/kibana_spec.rb b/spec/classes/module/kibana_spec.rb new file mode 100644 index 00000000..a28b7528 --- /dev/null +++ b/spec/classes/module/kibana_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::kibana' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-kibana').with_content( + %r{- module: kibana\n\s{2}log:\n\s{4}enabled: false\n\s{2}audit:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log and audit enabled with paths' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/kibana.log'], + 'audit_enabled' => true, + 'audit_paths' => ['/var/log/kibana-audit.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-kibana').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: kibana + log: + enabled: true + var.paths: + - "/var/log/kibana.log" + audit: + enabled: true + var.paths: + - "/var/log/kibana-audit.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/logstash_spec.rb b/spec/classes/module/logstash_spec.rb new file mode 100644 index 00000000..2200d64d --- /dev/null +++ b/spec/classes/module/logstash_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::logstash' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-logstash').with_content( + %r{- module: logstash\n\s{2}log:\n\s{4}enabled: false\n\s{2}slowlog:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log and slowlog enabled with paths' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/logstash.log'], + 'slowlog_enabled' => true, + 'slowlog_paths' => ['/var/log/logstash-slowlog.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-logstash').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: logstash + log: + enabled: true + var.paths: + - "/var/log/logstash.log" + slowlog: + enabled: true + var.paths: + - "/var/log/logstash-slowlog.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/mysql_spec.rb b/spec/classes/module/mysql_spec.rb new file mode 100644 index 00000000..25bc0781 --- /dev/null +++ b/spec/classes/module/mysql_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::mysql' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-mysql').with_content( + %r{- module: mysql\n\s{2}error:\n\s{4}enabled: false\n\s{2}slowlog:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on error and slowlog enabled with paths' do + let(:params) do + { + 'error_enabled' => true, + 'error_paths' => ['/var/log/mysql/error.log'], + 'slowlog_enabled' => true, + 'slowlog_paths' => ['/var/log/mysql/slowlog.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-mysql').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: mysql + error: + enabled: true + var.paths: + - "/var/log/mysql/error.log" + slowlog: + enabled: true + var.paths: + - "/var/log/mysql/slowlog.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/nginx_spec.rb b/spec/classes/module/nginx_spec.rb new file mode 100644 index 00000000..4a2a6111 --- /dev/null +++ b/spec/classes/module/nginx_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::nginx' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-nginx').with_content( + %r{- module: nginx\n\s{2}access:\n\s{4}enabled: false\n\s{2}error:\n\s{4}enabled: false\n\s{2}ingress_controller:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on access, error and ingress_controller enabled with paths' do + let(:params) do + { + 'access_enabled' => true, + 'access_paths' => ['/var/log/nginx/access.log'], + 'error_enabled' => true, + 'error_paths' => ['/var/log/nginx/error.log'], + 'ingress_controller_enabled' => true, + 'ingress_controller_paths' => ['/var/log/nginx/ingress_controller.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-nginx').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: nginx + access: + enabled: true + var.paths: + - "/var/log/nginx/access.log" + error: + enabled: true + var.paths: + - "/var/log/nginx/error.log" + ingress_controller: + enabled: true + var.paths: + - "/var/log/nginx/ingress_controller.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/postgresql_spec.rb b/spec/classes/module/postgresql_spec.rb new file mode 100644 index 00000000..8ad41728 --- /dev/null +++ b/spec/classes/module/postgresql_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::postgresql' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-postgresql').with_content( + %r{- module: postgresql\n\s{2}log:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log enabled with paths' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/postgresql.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-postgresql').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: postgresql + log: + enabled: true + var.paths: + - "/var/log/postgresql.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/rabbitmq_spec.rb b/spec/classes/module/rabbitmq_spec.rb new file mode 100644 index 00000000..6aa4c5e7 --- /dev/null +++ b/spec/classes/module/rabbitmq_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::rabbitmq' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-rabbitmq').with_content( + %r{- module: rabbitmq\n\s{2}log:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log enabled with paths' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/rabbitmq.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-rabbitmq').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: rabbitmq + log: + enabled: true + var.paths: + - "/var/log/rabbitmq.log" + +EOS + ) + } + end +end diff --git a/spec/classes/module/redis_spec.rb b/spec/classes/module/redis_spec.rb new file mode 100644 index 00000000..967ceb2d --- /dev/null +++ b/spec/classes/module/redis_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::redis' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-redis').with_content( + %r{- module: redis\n\s{2}log:\n\s{4}enabled: false\n\s{2}slowlog:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log and slowlog enabled with paths' do + let(:params) do + { + 'log_enabled' => true, + 'log_paths' => ['/var/log/redis.log'], + 'slowlog_enabled' => true, + 'slowlog_hosts' => ['localhost:6379'], + 'slowlog_password' => 'password', + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-redis').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: redis + log: + enabled: true + var.paths: + - "/var/log/redis.log" + slowlog: + enabled: true + var.hosts: + - localhost:6379 + var.password: password + +EOS + ) + } + end +end diff --git a/spec/classes/module/sophos_spec.rb b/spec/classes/module/sophos_spec.rb new file mode 100644 index 00000000..c3cb9a0a --- /dev/null +++ b/spec/classes/module/sophos_spec.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::sophos' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-sophos').with_content( + %r{- module: sophos\n\s{2}xg:\n\s{4}enabled: false\n\s{2}utm:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on xg and utm enabled with paths' do + let(:params) do + { + 'xg_enabled' => true, + 'xg_input' => 'file', + 'xg_paths' => ['/var/log/xg.log'], + 'utm_enabled' => true, + 'utm_input' => 'file', + 'utm_paths' => ['/var/log/utm.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-sophos').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: sophos + xg: + enabled: true + var.input: file + var.paths: + - "/var/log/xg.log" + utm: + enabled: true + var.input: file + var.paths: + - "/var/log/utm.log" + +EOS + ) + } + end + + context 'on xg and utm enabled with syslog input' do + let(:params) do + { + 'xg_enabled' => true, + 'xg_input' => 'udp', + 'xg_syslog_host' => '0.0.0.0', + 'xg_syslog_port' => 514, + 'xg_host_name' => 'sophos-xg', + 'utm_enabled' => true, + 'utm_input' => 'tcp', + 'utm_syslog_host' => '0.0.0.0', + 'utm_syslog_port' => 515, + 'utm_tz_offset' => '-07:00', + 'utm_rsa_fields' => true, + 'utm_keep_raw_fields' => true, + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-sophos').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: sophos + xg: + enabled: true + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 514 + var.host_name: sophos-xg + utm: + enabled: true + var.input: tcp + var.syslog_host: 0.0.0.0 + var.syslog_port: 515 + var.tz_offset: "-07:00" + var.rsa_fields: true + var.keep_raw_fields: true + +EOS + ) + } + end +end diff --git a/spec/classes/module/system_spec.rb b/spec/classes/module/system_spec.rb new file mode 100644 index 00000000..c52397fa --- /dev/null +++ b/spec/classes/module/system_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module::system' do + let :pre_condition do + 'include ::filebeat' + end + + let(:facts) { + { + :kernel => 'Linux', + :os => { + :family => 'Debian', + :name => 'Ubuntu', + } + } + } + + context 'on default values' do + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-system').with_content( + %r{- module: system\n\s{2}syslog:\n\s{4}enabled: false\n\s{2}auth:\n\s{4}enabled: false\n\n}, + )} + end + + context 'on log and slowlog enabled with paths' do + let(:params) do + { + 'syslog_enabled' => true, + 'syslog_paths' => ['/var/log/syslog'], + 'auth_enabled' => true, + 'auth_paths' => ['/var/log/auth.log'], + } + end + + it { is_expected.to compile.with_all_deps } + + it { + is_expected.to contain_file('filebeat-module-system').with_content( + <<-EOS +### Filebeat configuration managed by Puppet ### +--- +- module: system + syslog: + enabled: true + var.paths: + - "/var/log/syslog" + auth: + enabled: true + var.paths: + - "/var/log/auth.log" + +EOS + ) + } + end +end diff --git a/spec/classes/params_spec.rb b/spec/classes/params_spec.rb index f509e153..ea48df85 100644 --- a/spec/classes/params_spec.rb +++ b/spec/classes/params_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe 'filebeat::params' do - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } diff --git a/spec/classes/repo_spec.rb b/spec/classes/repo_spec.rb index 75a59e40..96421792 100644 --- a/spec/classes/repo_spec.rb +++ b/spec/classes/repo_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe 'filebeat::repo' do - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}' }" } @@ -13,7 +13,7 @@ case os_facts[:kernel] when 'Linux' it { is_expected.to compile } unless os_facts[:os]['family'] == 'Archlinux' - case os_facts[:osfamily] + case os_facts[:os]['family'] when 'Debian' it { is_expected.to contain_apt__source('beats').with( diff --git a/spec/classes/service_spec.rb b/spec/classes/service_spec.rb index a8292355..4b611a71 100644 --- a/spec/classes/service_spec.rb +++ b/spec/classes/service_spec.rb @@ -5,7 +5,7 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os.each do |os, os_facts| context "on #{os}" do let(:facts) { os_facts } diff --git a/spec/default_facts.yml b/spec/default_facts.yml index 3248be5a..f777abfc 100644 --- a/spec/default_facts.yml +++ b/spec/default_facts.yml @@ -2,7 +2,7 @@ # # Facts specified here will override the values provided by rspec-puppet-facts. --- -concat_basedir: "/tmp" ipaddress: "172.16.254.254" +ipaddress6: "FE80:0000:0000:0000:AAAA:AAAA:AAAA" is_pe: false macaddress: "AA:AA:AA:AA:AA:AA" diff --git a/spec/default_module_facts.yml b/spec/default_module_facts.yml new file mode 100644 index 00000000..eaf98fb4 --- /dev/null +++ b/spec/default_module_facts.yml @@ -0,0 +1,3 @@ +filebeat_version: undef +concat_basedir: "/tmp" +service_provider: systemd \ No newline at end of file diff --git a/spec/defines/input_spec.rb b/spec/defines/input_spec.rb new file mode 100644 index 00000000..27f8f287 --- /dev/null +++ b/spec/defines/input_spec.rb @@ -0,0 +1,176 @@ +require 'spec_helper' + +describe 'filebeat::input' do + let :pre_condition do + 'class { "filebeat": + outputs => { + "logstash" => { + "hosts" => [ + "localhost:5044", + ], + }, + }, + inputs => [ + { + "type" => "logs", + "paths" => [ + "/var/log/auth.log", + "/var/log/syslog", + ], + }, + { + "type" => "syslog", + "protocol.tcp" => { + "host" => "0.0.0.0:514", + }, + }, + ], + }' + end + + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + let(:title) { 'test-logs' } + let(:params) do + { + 'paths' => [ + '/var/log/auth.log', + '/var/log/syslog', + ], + 'doc_type' => 'syslog-beat', + } + end + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { + is_expected.to contain_file('filebeat-test-logs').with( + notify: 'Service[filebeat]', + ) + } + end + + context "with docker input support on #{os}" do + let(:facts) { os_facts } + + # Docker Support + let(:title) { 'docker' } + let(:params) do + { + 'input_type' => 'docker', + } + end + + if os_facts[:kernel] == 'Linux' + it { is_expected.to compile } + + it { + is_expected.to contain_file('filebeat-docker').with( + notify: 'Service[filebeat]', + ) + is_expected.to contain_file('filebeat-docker').with_content( + %r{- type: docker\n\s{2}containers:\n\s{4}ids:\n\s{4}- '\*'\n\s{4}path: /var/lib/docker/containers\n\s{4}stream: all\n\s{2}combine_partial: false\n\s{2}cri.parse_flags: false\n}, + ) + } + end + end + + context "with filestream input support on #{os}" do + let(:facts) { os_facts } + + # Filestream + let(:title) { 'some-filestream' } + + context "with take_over unset" do + let(:params) do + { + 'input_type' => 'filestream', + 'paths' => ['/var/log/foo.log'], + } + end + + if os_facts[:kernel] == 'Linux' + it { is_expected.to compile } + + it { + is_expected.to contain_file('filebeat-some-filestream').with( + notify: 'Service[filebeat]', + ) + is_expected.to contain_file('filebeat-some-filestream').with_content( + %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}paths:\n\s{2}- /var/log/foo.log}, + ) + } + end + end + + context "with take_over => true" do + let(:params) do + { + 'input_type' => 'filestream', + 'paths' => ['/var/log/foo.log'], + 'take_over' => true, + } + end + + if os_facts[:kernel] == 'Linux' + it { is_expected.to compile } + + it { + is_expected.to contain_file('filebeat-some-filestream').with( + notify: 'Service[filebeat]', + ) + is_expected.to contain_file('filebeat-some-filestream').with_content( + %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}take_over: true\n\s{2}paths:\n\s{2}- /var/log/foo.log}, + ) + } + end + end + end + end + + on_supported_os.each do |os, os_facts| + context "with array input support on #{os}" do + let(:facts) { os_facts } + + # Docker Support + let(:title) { 'test-array' } + let(:params) do + { + 'pure_array' => true, + } + end + + if os_facts[:kernel] == 'Linux' + it { is_expected.to compile } + + it { + is_expected.to contain_file('filebeat-test-array').with( + notify: 'Service[filebeat]', + ) + is_expected.to contain_file('filebeat-test-array').with_content( + %r{- type: logs\n\s{2}paths:\n\s{2}- "/var/log/auth.log"\n\s{2}- "/var/log/syslog"\n- type: syslog\n\s{2}protocol.tcp:\n\s{4}host: 0.0.0.0:514\n}, + ) + } + end + end + end + + context 'with no parameters' do + let(:title) { 'test-logs' } + let(:params) do + { + 'paths' => [ + '/var/log/auth.log', + '/var/log/syslog', + ], + 'doc_type' => 'syslog-beat', + } + end + + it { is_expected.to raise_error(Puppet::Error) } + end +end diff --git a/spec/defines/module_spec.rb b/spec/defines/module_spec.rb new file mode 100644 index 00000000..4b9194fc --- /dev/null +++ b/spec/defines/module_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'filebeat::module' do + let :pre_condition do + 'class { "filebeat": + outputs => { + "logstash" => { + "hosts" => [ + "localhost:5044", + ], + }, + }, + inputs => [ + { + "type" => "logs", + "paths" => [ + "/var/log/auth.log", + "/var/log/syslog", + ], + }, + { + "type" => "syslog", + "protocol.tcp" => { + "host" => "0.0.0.0:514", + }, + }, + ], + }' + end + + let(:title) { 'test' } + let(:params) do + {} + end + + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + if os_facts[:kernel] == 'Linux' + it { is_expected.to compile } + + it { + is_expected.to contain_file('filebeat-module-test').with( + notify: 'Service[filebeat]', + ) + is_expected.to contain_file('filebeat-module-test').with_content( + %r{- module: test}, + ) + } + end + end + end +end diff --git a/spec/defines/prospector_spec.rb b/spec/defines/prospector_spec.rb deleted file mode 100644 index 56429f07..00000000 --- a/spec/defines/prospector_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -require 'spec_helper' - -describe 'filebeat::prospector' do - let :pre_condition do - 'class { "filebeat": - outputs => { - "logstash" => { - "hosts" => [ - "localhost:5044", - ], - }, - }, - }' - end - - let(:title) { 'test-logs' } - let(:params) do - { - 'paths' => [ - '/var/log/auth.log', - '/var/log/syslog', - ], - 'doc_type' => 'syslog-beat', - } - end - - on_supported_os(facterversion: '2.4').each do |os, os_facts| - context "on #{os}" do - let(:facts) { os_facts } - - if os_facts[:kernel] != 'windows' - it { is_expected.to compile } - end - - it { - is_expected.to contain_file('filebeat-test-logs').with( - notify: 'Service[filebeat]', - ) - } - end - end - - context 'with no parameters' do - it { is_expected.to raise_error(Puppet::Error) } - end -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 15266c20..7d864544 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,14 @@ +# frozen_string_literal: true + +RSpec.configure do |c| + c.mock_with :mocha +end + require 'puppetlabs_spec_helper/module_spec_helper' require 'rspec-puppet-facts' + +require 'spec_helper_local' if File.file?(File.join(File.dirname(__FILE__), 'spec_helper_local.rb')) + include RspecPuppetFacts default_facts = { @@ -7,17 +16,58 @@ facterversion: Facter.version, } -default_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')) -default_module_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')) +default_fact_files = [ + File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')), + File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')), +] -if File.exist?(default_facts_path) && File.readable?(default_facts_path) - default_facts.merge!(YAML.safe_load(File.read(default_facts_path))) +default_fact_files.each do |f| + next unless File.exist?(f) && File.readable?(f) && File.size?(f) + + begin + default_facts.merge!(YAML.safe_load(File.read(f), [], [], true)) + rescue => e + RSpec.configuration.reporter.message "WARNING: Unable to load #{f}: #{e}" + end end -if File.exist?(default_module_facts_path) && File.readable?(default_module_facts_path) - default_facts.merge!(YAML.safe_load(File.read(default_module_facts_path))) +# read default_facts and merge them over what is provided by facterdb +default_facts.each do |fact, value| + add_custom_fact fact, value end RSpec.configure do |c| c.default_facts = default_facts + c.before :each do + # set to strictest setting for testing + # by default Puppet runs at warning level + Puppet.settings[:strict] = :warning + Puppet.settings[:strict_variables] = true + end + c.filter_run_excluding(bolt: true) unless ENV['GEM_BOLT'] + c.after(:suite) do + end + + # Filter backtrace noise + backtrace_exclusion_patterns = [ + %r{spec_helper}, + %r{gems}, + ] + + if c.respond_to?(:backtrace_exclusion_patterns) + c.backtrace_exclusion_patterns = backtrace_exclusion_patterns + elsif c.respond_to?(:backtrace_clean_patterns) + c.backtrace_clean_patterns = backtrace_exclusion_patterns + end end + +# Ensures that a module is defined +# @param module_name Name of the module +def ensure_module_defined(module_name) + module_name.split('::').reduce(Object) do |last_module, next_module| + last_module.const_set(next_module, Module.new) unless last_module.const_defined?(next_module, false) + last_module.const_get(next_module, false) + end +end + +# 'spec_overrides' from sync.yml will appear below this line diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index b8bbd361..bfc8d47b 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -7,7 +7,7 @@ install_module_on(hosts) install_module_dependencies_on(hosts) -UNSUPPORTED_PLATFORMS = %w[aix Solaris BSD].freeze +UNSUPPORTED_PLATFORMS = ['aix', 'Solaris', 'BSD'].freeze RSpec.configure do |c| module_root = File.expand_path(File.join(File.dirname(__FILE__), '..')) diff --git a/spec/unit/facter/filebeat_version_spec.rb b/spec/unit/facter/filebeat_version_spec.rb index 4f3bc141..f20fdf52 100644 --- a/spec/unit/facter/filebeat_version_spec.rb +++ b/spec/unit/facter/filebeat_version_spec.rb @@ -16,4 +16,20 @@ expect(Facter.fact(:filebeat_version).value).to eq('5.1.1') end end + + context 'when the filebeat package is not installed' do + before :each do + File.stubs(:executable?) + Facter::Util::Resolution.stubs(:exec) + File.expects(:executable?).with('/usr/bin/filebeat').returns false + File.expects(:executable?).with('/usr/local/bin/filebeat').returns false + File.expects(:executable?).with('/usr/share/filebeat/bin/filebeat').returns false + File.expects(:executable?).with('/usr/local/sbin/filebeat').returns false + File.stubs(:exist?) + File.expects(:exist?).with('c:\Program Files\Filebeat\filebeat.exe').returns false + end + it 'returns false' do + expect(Facter.fact(:filebeat_version).value).to eq(false) + end + end end diff --git a/templates/filebeat.yml.erb b/templates/filebeat.yml.erb index b063c27d..e0b66f7c 100644 --- a/templates/filebeat.yml.erb +++ b/templates/filebeat.yml.erb @@ -5,9 +5,12 @@ filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %> filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %> filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %> <% end -%> -filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %> filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %> +filebeat.overwrite_pipelines: <%= @filebeat_config['filebeat']['overwrite_pipelines'] %> filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %> +filebeat.registry.path: <%= @filebeat_config['filebeat']['registry']['path'] %> +filebeat.registry.file_permissions: <%= @filebeat_config['filebeat']['registry']['file_permissions'] %> +filebeat.registry.flush: <%= @filebeat_config['filebeat']['registry']['flush'] %> #================================ General ===================================== diff --git a/templates/input.yml.erb b/templates/input.yml.erb new file mode 100644 index 00000000..87126ae9 --- /dev/null +++ b/templates/input.yml.erb @@ -0,0 +1,319 @@ +<%- if @pure_array -%> +<%= scope['filebeat::inputs'].to_yaml() %> +<%- else -%> +--- +- type: <%= @input_type %> + <%- if @input_type =~ /(filestream|journald)/ -%> + id: <%= @name %> + <%- end -%> + <%- if @input_type == 'filestream' && @take_over != nil && @take_over == true -%> + take_over: true + <%- end -%> + <%- if @input_type =~ /(tcp|udp)/ -%> + host: <%= @host %> + <%- if @ssl.length > 0 -%> + ssl: + <%- @ssl.each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + <%- if @max_message_size -%> + max_message_size: <%= @max_message_size %> + <%- end -%> + <%- elsif @input_type == 'docker' -%> + containers: + ids: + <%- @containers_ids.each do |id| -%> + - <%= id %> + <%- end -%> + path: <%= @containers_path %> + stream: <%= @containers_stream %> + combine_partial: <%= @combine_partial %> + cri.parse_flags: <%= @cri_parse_flags %> + <%- elsif @input_type == 'syslog' -%> + protocol.<%= @syslog_protocol %>: + host: <%= @syslog_host %> + <%- if @ssl.length > 0 -%> + ssl: + <%- @ssl.each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + <%- elsif @input_type == 'journald' -%> + <%- if @seek -%> + seek: <%= @seek %> + <%- end -%> + <%- if @include_matches.length > 0 -%> + include_matches: + <%- @include_matches.each do |match| -%> + - <%= match %> + <%- end -%> + <%- end -%> + <%- else -%> + paths: + <%- @paths.each do |log_path| -%> + - <%= log_path %> + <%- end -%> + <%- if @encoding -%> + encoding: <%= @encoding %> + <%- end -%> + <%- if @include_lines.length > 0 -%> + include_lines: + <%- @include_lines.each do |include_line| -%> + - '<%= include_line %>' + <%- end -%> + <%- end -%> + <%- if @exclude_lines.length > 0 -%> + exclude_lines: + <%- @exclude_lines.each do |exclude_line| -%> + - '<%= exclude_line %>' + <%- end -%> + <%- end -%> + <%- if @input_type != 'filestream' -%> + <%- if @exclude_files.length > 0 -%> + exclude_files: + <%- @exclude_files.each do |exclude_file| -%> + - <%= exclude_file %> + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @ignore_older -%> + ignore_older: <%= @ignore_older %> + <%- end -%> + <%- if @doc_type -%> + document_type: <%= @doc_type %> + <%- end -%> + <%- end -%> + <%- if @scan_frequency or @exclude_files -%> + <%- if @input_type == 'filestream' -%> + prospector: + scanner: + <%- if @scan_frequency -%> + check_interval: <%= @scan_frequency %> + <%- end -%> + <%- if @exclude_files.length > 0 -%> + exclude_files: + <%- @exclude_files.each do |exclude_file| -%> + - <%= exclude_file %> + <%- end -%> + <%- end -%> + <%- else -%> + scan_frequency: <%= @scan_frequency %> + <%- end -%> + <%- end -%> + <%- if @harvester_buffer_size -%> + harvester_buffer_size: <%= @harvester_buffer_size %> + <%- end -%> + <%- if @max_bytes -%> + <%- if @input_type == 'filestream' -%> + message_max_bytes: <%= @max_bytes %> + <%- else -%> + max_bytes: <%= @max_bytes %> + <%- end -%> + <%- end -%> + <%- if @symlinks -%> + symlinks: <%= @symlinks %> + <%- end -%> + <%- if @close_older -%> + close_older: <%= @close_older %> + <%- end -%> + <%- if @force_close_files -%> + force_close_files: <%= @force_close_files %> + <%- end -%> + <%- if @json.length > 0 -%> + ###JSON configuration + json: + # Decode JSON options. Enable this if your logs are structured in JSON. + # JSON key on which to apply the line filtering and multiline settings. This key + # must be top level and its value must be string, otherwise it is ignored. If + # no text key is defined, the line filtering and multiline features cannot be used. + <%- if @json['message_key'] != nil -%> + message_key: '<%= @json['message_key'] %>' + <%- end -%> + + # By default, the decoded JSON is placed under a "json" key in the output document. + # If you enable this setting, the keys are copied top level in the output document. + <%- if @json['keys_under_root'] != nil -%> + keys_under_root: <%= @json['keys_under_root'] %> + <%- end -%> + + # If keys_under_root and this setting are enabled, then the values from the decoded + # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) + # in case of conflicts. + <%- if @json['overwrite_keys'] != nil -%> + overwrite_keys: <%= @json['overwrite_keys'] %> + <%- end -%> + + # If this setting is enabled, Filebeat will recursively de-dot keys in the decoded JSON, + # and expand them into a hierarchical object structure. For example, {"a.b.c": 123} + # would be expanded into {"a":{"b":{"c":123}}}. This setting should be enabled when + # the input is produced by an ECS logger. + <%- if @json['expand_keys'] != nil -%> + expand_keys: <%= @json['expand_keys'] %> + <%- end -%> + + # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON + # unmarshaling errors or when a text key is defined in the configuration but cannot + # be used. + <%- if @json['add_error_key'] != nil -%> + add_error_key: <%= @json['add_error_key'] %> + <%- end -%> + + # Optional configuration setting that specifies the JSON key to set the document id. + # If configured, the field will be removed from the original json document and + # stored in @metadata._id + <%- if @json['document_id'] != nil -%> + document_id: <%= @json['document_id'] %> + <%- end -%> + + # An optional configuration setting that specifies if JSON decoding errors should + # be logged or not. If set to true, errors will not be logged. The default is false. + <%- if @json['ignore_decoding_error'] != nil -%> + ignore_decoding_error: <%= @json['ignore_decoding_error'] %> + <%- end -%> + <%- end -%> + <%- if @multiline.length > 0 -%> + <%- if @input_type == 'filestream' -%> + parsers: + - multiline: + <%- if @multiline['pattern'] -%> + pattern: '<%= @multiline['pattern'] %>' + <%- end -%> + <%- if @multiline['negate'] -%> + negate: <%= @multiline['negate'] %> + <%- end -%> + <%- if @multiline['match'] -%> + match: <%= @multiline['match'] %> + <%- end -%> + <%- if @multiline['max_lines'] -%> + max_lines: <%= @multiline['max_lines'] %> + <%- end -%> + <%- if @multiline['timeout'] -%> + timeout: <%= @multiline['timeout'] %> + <%- end -%> + <%- else -%> + multiline: + <%- if @multiline['pattern'] -%> + pattern: '<%= @multiline['pattern'] %>' + <%- end -%> + <%- if @multiline['negate'] -%> + negate: <%= @multiline['negate'] %> + <%- end -%> + <%- if @multiline['match'] -%> + match: <%= @multiline['match'] %> + <%- end -%> + <%- if @multiline['max_lines'] -%> + max_lines: <%= @multiline['max_lines'] %> + <%- end -%> + <%- if @multiline['timeout'] -%> + timeout: <%= @multiline['timeout'] %> + <%- end -%> + <%- end -%> + <%- end -%> + tail_files: <%= @tail_files %> + + # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the + # original for harvesting but will report the symlink name as source. + #symlinks: false + + <%- if @backoff -%> + <%- if @input_type == 'filestream' -%> + backoff.init: <%= @backoff %> + <%- else -%> + backoff: <%= @backoff %> + <%- end -%> + <%- end -%> + <%- if @max_backoff -%> + <%- if @input_type == 'filestream' -%> + backoff.max: <%= @max_backoff %> + <%- else -%> + max_backoff: <%= @max_backoff %> + <%- end -%> + <%- end -%> + <%- if @backoff_factor -%> + backoff_factor: <%= @backoff_factor %> + <%- end -%> + + # Experimental: Max number of harvesters that are started in parallel. + # Default is 0 which means unlimited + <%- if @harvester_limit -%> + harvester_limit: <%= @harvester_limit %> + <%- end -%> + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + <%- if @close_inactive -%> + close_inactive: <%= @close_inactive %> + <%- end -%> + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_renamed: <%= @close_renamed %> + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + close_removed: <%= @close_removed %> + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_eof: <%= @close_eof %> + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + <%- if @clean_inactive -%> + clean_inactive: <%= @clean_inactive %> + <%- end -%> + + # Removes the state for file which cannot be found on disk anymore immediately + clean_removed: <%= @clean_removed %> + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + <%- if @close_timeout -%> + close_timeout: <%= @close_timeout %> + <%- end -%> + <%- # Everything below this can be applied to any input. -%> + <%- # https://www.elastic.co/guide/en/beats/filebeat/current/configuration-general-options.html#configuration-general -%> + <%- if @pipeline -%> + pipeline: <%= @pipeline %> + <%- end -%> + <%- if @fields.length > 0 -%> + fields: + <%- @fields.each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + fields_under_root: <%= @fields_under_root %> + <%- unless @facts['filebeat_version'] == false -%> + <%- if scope.function_versioncmp([@facts['filebeat_version'], '7.5']) > 0 -%> + <%- if @index -%> + index: <%= @index %> + <%- end -%> + <%- if @keep_null -%> + keep_null: <%= @keep_null %> + <%- end -%> + <%- end -%> + <%- if scope.function_versioncmp([@facts['filebeat_version'], '7.9']) > 0 -%> + publisher_pipeline.disable_host: <%= @publisher_pipeline_disable_host %> + <%- end -%> + <%- end -%> + <%- if @tags.length > 0 -%> + tags: + <%- @tags.each do |tag| -%> + - <%= tag %> + <%- end -%> + <%- end -%> + <%- if @processors.length > 0 -%> + processors: + <%- %><%= @processors.to_yaml.lines.drop(1).join.gsub(/^/, ' ') -%> + <%- end -%> +<%- end %> diff --git a/templates/prospector.yml.erb b/templates/prospector.yml.erb index 1d1fcfab..3462fc8e 100644 --- a/templates/prospector.yml.erb +++ b/templates/prospector.yml.erb @@ -98,6 +98,12 @@ filebeat: <%- if @json['add_error_key'] != nil -%> add_error_key: <%= @json['add_error_key'] %> <%- end -%> + + # An optional configuration setting that specifies if JSON decoding errors should + # be logged or not. If set to true, errors will not be logged. The default is false. + <%- if @json['ignore_decoding_error'] != nil -%> + ignore_decoding_error: <%= @json['ignore_decoding_error'] %> + <%- end -%> <%- end -%> <%- if @multiline.length > 0 -%> @@ -136,7 +142,9 @@ filebeat: # Experimental: Max number of harvesters that are started in parallel. # Default is 0 which means unlimited - #harvester_limit: 0 + <%- if @harvester_limit -%> + harvester_limit: <%= @harvester_limit %> + <%- end -%> ### Harvester closing options diff --git a/templates/systemd/logging.conf.erb b/templates/systemd/logging.conf.erb new file mode 100644 index 00000000..42762f98 --- /dev/null +++ b/templates/systemd/logging.conf.erb @@ -0,0 +1,2 @@ +[Service] +Environment="BEAT_LOG_OPTS=<%= @systemd_beat_log_opts_override %>" \ No newline at end of file