Skip to content

Commit

Permalink
refactor: delete brod_cli code
Browse files Browse the repository at this point in the history
  • Loading branch information
zmstone committed Oct 5, 2024
1 parent 494a0cc commit 777e697
Show file tree
Hide file tree
Showing 12 changed files with 2 additions and 2,336 deletions.
2 changes: 0 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ jobs:
rebar3-version: ${{ env.REBAR_VERSION }}
- name: Compile
run: rebar3 do compile
- name: Make brod_cli script
run: rebar3 as brod_cli escriptize
- name: Run tests
run: |
export KAFKA_VERSION=${{ matrix.kafka }}
Expand Down
7 changes: 0 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,6 @@ hex-publish: clean
## tests that require kafka running at localhost
INTEGRATION_CTS = brod_cg_commits brod_client brod_compression brod_consumer brod_producer brod_group_subscriber brod_topic_subscriber brod

## build escript and a release, and copy escript to release bin dir
brod-cli:
@rebar3 as brod_cli do compile,escriptize,release
@cp _build/brod_cli/bin/brod_cli _build/brod_cli/rel/brod/bin/
@cp scripts/brod _build/brod_cli/rel/brod/bin/
@cp scripts/brod.escript _build/brod_cli/rel/brod/bin/

cover:
@rebar3 cover -v

Expand Down
99 changes: 0 additions & 99 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -490,102 +490,3 @@ brod:delete_topics(Hosts, [Topic], Timeout).
```

Caution the above delete_topics can fail if you do not have `delete.topic.enable` set to true in your kafka config

## brod-cli: A command line tool to interact with Kafka

This will build a self-contained binary with brod application

```bash
make brod-cli
_build/brod_cli/rel/brod/bin/brod -h
```

Disclaimer: This script is NOT designed for use cases where fault-tolerance is a hard requirement.
As it may crash when e.g. kafka cluster is temporarily unreachable,
or (for fetch command) when the partition leader migrates to another broker in the cluster.

### brod-cli examples (with `alias brod=_build/brod_cli/rel/brod/bin/brod`):

#### Fetch and print metadata

```sh
brod meta -b localhost
```

#### Produce a Message

```sh
brod send -b localhost -t test-topic -p 0 -k "key" -v "value"

```

#### Fetch a Message

```sh
brod fetch -b localhost -t test-topic -p 0 --fmt 'io:format("offset=~p, ts=~p, key=~s, value=~s\n", [Offset, Ts, Key, Value])'
```

Bound variables to be used in `--fmt` expression:

- `Offset`: Message offset
- `Key`: Kafka key
- `Value`: Kafka Value
- `TsType`: Timestamp type either `create` or `append`
- `Ts`: Timestamp, `-1` as no value

#### Stream Messages to Kafka

Send `README.md` to kafka one line per kafka message

```sh
brod pipe -b localhost:9092 -t test-topic -p 0 -s @./README.md
```

#### Resolve Offset

```sh
brod offset -b localhost:9092 -t test-topic -p 0
```

#### List or Describe Groups

```sh
# List all groups
brod groups -b localhost:9092

# Describe groups
brod groups -b localhost:9092 --ids group-1,group-2
```

#### Display Committed Offsets

```sh
# all topics
brod commits -b localhost:9092 --id the-group-id --describe

# a specific topic
brod commits -b localhost:9092 --id the-group-id --describe --topic topic-name
```

#### Commit Offsets

NOTE: This feature is designed for force overwriting commits, not for regular use of offset commit.

```sh
# Commit 'latest' offsets of all partitions with 2 days retention
brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 2d

# Commit offset=100 for partition 0 and 200 for partition 1
brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets "0:100,1:200"

# Use --retention 0 to delete commits (may linger in kafka before cleaner does its job)
brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 0

# Try join an active consumer group using 'range' protocol and steal one partition assignment then commit offset=10000
brod commits -b localhost:9092 -i the-group-id -t topic-name -o "0:10000" --protocol range
```

### TODOs

- Support scram-sasl in brod-cli
- Transactional produce APIs
2 changes: 0 additions & 2 deletions elvis.config
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@
#{ level => 3,
ignore => [ brod_group_coordinator
, brod_utils
, brod_cli
, brod_cli_pipe
]
}}
, {elvis_style, god_modules,
Expand Down
30 changes: 1 addition & 29 deletions rebar.config
Original file line number Diff line number Diff line change
@@ -1,38 +1,10 @@
{deps, [{kafka_protocol, "4.1.9"}]}.
{project_plugins, [{rebar3_lint, "~> 3.2.5"}]}.
{edoc_opts, [{preprocess, true}, {macros, [{build_brod_cli, true}]}]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warnings_as_errors, warn_unused_vars,warn_shadow_vars,warn_obsolete_guard,debug_info]}.
{xref_checks, [undefined_function_calls, undefined_functions,
locals_not_used, deprecated_function_calls,
deprecated_functions]}.
{profiles, [
{brod_cli, [
{deps, [ {docopt, {git, "https://github.com/zmstone/docopt-erl.git", {tag, "0.1.3"}}}
, {jsone, "1.7.0"}
, {snappyer, "1.2.9"}
]},
{erl_opts, [warnings_as_errors, {d, build_brod_cli}]},
{escript_name, brod_cli},
{relx, [{release, {brod, "i"}, % release the interactive shell as brod-i
[brod, jsone, docopt]},
{include_erts, true},
{overlay, [{copy, "scripts/brod", "bin/"},
{copy, "{{lib_dirs}}/crc32cer/priv/crc32cer*.so", "bin/"},
{copy, "{{lib_dirs}}/snappyer/priv/snappyer.so", "bin/"}
]}
]}]},
{test, [
{deps, [ {docopt, {git, "https://github.com/zmstone/docopt-erl.git", {tag, "0.1.3"}}}
, {hut, "1.3.0"}
, {jsone, "1.7.0"}
, {meck, "0.9.2"}
, {proper, "1.4.0"}
, {snappyer, "1.2.9"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {branch, "1.0.10"}}}
]},
{erl_opts, [warnings_as_errors, {d, build_brod_cli}]}
]}
]}.
{ex_doc,
[ {extras,
[ {"CHANGELOG.md", #{title => "Changelog"}}
Expand Down
32 changes: 0 additions & 32 deletions scripts/brod

This file was deleted.

31 changes: 0 additions & 31 deletions scripts/brod.escript

This file was deleted.

10 changes: 1 addition & 9 deletions src/brod.erl
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
, delete_topics/4
]).

%% APIs for quick metadata or message inspection and brod_cli
%% APIs for quick metadata or message inspection
-export([ get_metadata/1
, get_metadata/2
, get_metadata/3
Expand Down Expand Up @@ -128,10 +128,6 @@
, fetch/8
]).

-ifdef(build_brod_cli).
-export([main/1]).
-endif.

-export_type([ batch_input/0
, bootstrap/0
, call_ref/0
Expand Down Expand Up @@ -1365,10 +1361,6 @@ fetch_committed_offsets(BootstrapEndpoints, ConnCfg, GroupId) ->
fetch_committed_offsets(Client, GroupId) ->
brod_utils:fetch_committed_offsets(Client, GroupId, []).

-ifdef(build_brod_cli).
main(X) -> brod_cli:main(X).
-endif.

%% @doc Start a new transaction, `TxId' will be the id of the transaction
%% @equiv brod_transaction:start_link/3
-spec transaction(client(), transactional_id(), transaction_config()) -> {ok, transaction()}.
Expand Down
Loading

0 comments on commit 777e697

Please sign in to comment.