diff --git a/app/Makefile b/app/Makefile index a4c3b312..f18ee9ff 100644 --- a/app/Makefile +++ b/app/Makefile @@ -198,12 +198,14 @@ ZSV=$(BINDIR)/zsv${EXE} SOURCES=echo paste count count-pull select select-pull 2tsv 2json serialize flatten pretty stack desc sql 2db compare prop rm mv jq overwrite CLI_SOURCES=echo select desc count paste 2tsv pretty sql flatten 2json serialize stack 2db compare prop rm mv jq overwrite +SHEET_INCLUDES= ifeq ($(ZSVSHEET_BUILD),1) SOURCES+=sheet CLI_SOURCES+=sheet CFLAGS+=-DZSVSHEET_BUILD CFLAGS+=${CFLAGS_NCURSES} LDFLAGS+=${LDFLAGS_NCURSES} + SHEET_INCLUDES=$(wildcard sheet/*.c sheet/*.h) endif CFLAGS+= -DUSE_JQ @@ -285,6 +287,8 @@ SQLITE_SRC=${THIS_MAKEFILE_DIR}/external/sqlite3/sqlite3*.c SQLITE_EXT=${BUILD_DIR}-external/sqlite3/sqlite3_and_csv_vtab.o SQLITE_EXT_INCLUDE=-I${THIS_MAKEFILE_DIR}/external/sqlite3 +SQL_INTERNAL_OBJECT=${CLI_OBJ_PFX}sql_internal.o + # everything uses prop, which in turn uses yajl and jq and json and sqlite3 OBJECTS+= ${YAJL_OBJ} ${YAJL_HELPER_OBJ} ${BUILD_DIR}/objs/utils/json.o ${SQLITE_EXT} MORE_SOURCE+= ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -I${JQ_INCLUDE_DIR} ${SQLITE_EXT_INCLUDE} @@ -398,12 +402,18 @@ ${INIH_OBJECT}: ${INIH_SRC}/ini.c @mkdir -p `dirname "$@"` ${CC} ${CFLAGS} -I${INIH_INCLUDE} -DINI_HANDLER_LINENO=1 -DINI_CALL_HANDLER_ON_NEW_SECTION=1 -c $< -o $@ +${SQL_INTERNAL_OBJECT}: ${CLI_OBJ_PFX}%.o: %.c %.h + @mkdir -p `dirname "$@"` + ${CC} ${CFLAGS} -I${INCLUDE_DIR} -c -o $@ $< + +${CLI_OBJ_PFX}sheet.o: ${SHEET_INCLUDES} + ${CLI_APP_OBJECT} : cli_ini.c builtin/*.c ${JQ_LIB} ${CLI_APP_OBJECT} ${CLI_OBJECTS}: ${CLI_OBJ_PFX}%.o: %.c ${UTF8PROC_SRC}/utf8proc.c # ${MORE_OBJECTS} @mkdir -p `dirname "$@"` ${CC} ${CFLAGS} -DVERSION=\"${VERSION}\" -DZSV_CLI ${CLI_INCLUDE} -I${THIS_MAKEFILE_DIR}/external/sglib -I${INCLUDE_DIR} -I${UTF8PROC_INCLUDE} -c $< -o $@ ${MORE_SOURCE} -${CLI}: cli_internal.c.in cli_internal.h cli_internal.h.in ${CLI_APP_OBJECT} ${CLI_OBJECTS} ${OBJECTS} ${UTF8PROC_OBJECT} cli_ini.c ${INIH_OBJECT} ${LIBZSV_INSTALL} ${MORE_OBJECTS} +${CLI}: cli_internal.c.in cli_internal.h cli_internal.h.in ${CLI_APP_OBJECT} ${CLI_OBJECTS} ${OBJECTS} ${UTF8PROC_OBJECT} cli_ini.c ${INIH_OBJECT} ${LIBZSV_INSTALL} ${MORE_OBJECTS} ${SQL_INTERNAL_OBJECT} @mkdir -p `dirname "$@"` ${CC} ${CFLAGS} ${CFLAGS_EXE} -I${INCLUDE_DIR} -o $@ ${CLI_APP_OBJECT} ${CLI_OBJECTS} ${OBJECTS} ${UTF8PROC_OBJECT} ${INIH_OBJECT} -L${LIBDIR} ${LIBZSV_L} ${LDFLAGS} ${LDFLAGS_OPT} ${MORE_OBJECTS} ${MORE_SOURCE} ${MORE_LIBS} ${STATIC_LIB_FLAGS} @echo Built $@ @@ -468,6 +478,10 @@ ${CLI} ${STANDALONE_PFX}2json${EXE}: MORE_OBJECTS+= ${BUILD_DIR}/objs/utils/db.o # pretty uses termcap ${CLI} ${STANDALONE_PFX}pretty${EXE}: MORE_LIBS+=${LDFLAGS_TERMCAP} +${STANDALONE_PFX}sheet${EXE}: ${SHEET_INCLUDES} + +${CLI} ${STANDALONE_PFX}sheet${EXE} ${STANDALONE_PFX}sql${EXE}: ${SQL_INTERNAL_OBJECT} +${CLI} ${STANDALONE_PFX}sheet${EXE} ${STANDALONE_PFX}sql${EXE}: MORE_OBJECTS+=${SQL_INTERNAL_OBJECT} ${STANDALONE_PFX}%${EXE}: %.c ${OBJECTS} ${MORE_OBJECTS} ${LIBZSV_INSTALL} ${UTF8PROC_OBJECT} @mkdir -p `dirname "$@"` diff --git a/app/cli.c b/app/cli.c index cc71173b..1bd187f6 100644 --- a/app/cli.c +++ b/app/cli.c @@ -419,6 +419,7 @@ static struct zsv_ext_callbacks *zsv_ext_callbacks_init(struct zsv_ext_callbacks e->ext_sheet_buffer_filename = zsvsheet_buffer_filename; e->ext_sheet_buffer_data_filename = zsvsheet_buffer_data_filename; e->ext_sheet_open_file = zsvsheet_open_file; + e->ext_sheet_open_file_opts = zsvsheet_ext_open_file_opts; e->ext_sheet_register_proc = zsvsheet_register_proc; e->ext_sheet_register_proc_key_binding = zsvsheet_register_proc_key_binding; e->ext_sheet_push_transformation = zsvsheet_push_transformation; diff --git a/app/ext_example/mysheet_extension.c b/app/ext_example/mysheet_extension.c index 2111572c..60d31d01 100644 --- a/app/ext_example/mysheet_extension.c +++ b/app/ext_example/mysheet_extension.c @@ -189,9 +189,12 @@ static zsvsheet_status zsv_sqlite3_to_csv(zsvsheet_proc_context_t pctx, struct z if (writer_opts.stream) fclose(writer_opts.stream); - if (tmp_fn && zsv_file_exists(tmp_fn)) - zst = zsv_cb.ext_sheet_open_file(pctx, tmp_fn, NULL); - else { + if (tmp_fn && zsv_file_exists(tmp_fn)) { + struct zsvsheet_open_file_opts ofopts = {0}; + ofopts.data_filename = tmp_fn; + ofopts.no_auto_row_num = 1; + zst = zsv_cb.ext_sheet_open_file_opts(pctx, &ofopts); + } else { if (zst == zsvsheet_status_ok) { zst = zsvsheet_status_error; // to do: make this more specific if (!err_msg && zdb && zdb->rc != SQLITE_OK) @@ -229,12 +232,9 @@ zsvsheet_status pivot_drill_down(zsvsheet_proc_context_t ctx) { if (!zdb || !(sql_str = sqlite3_str_new(zdb->db))) zst = zsvsheet_status_memory; else if (zdb->rc == SQLITE_OK && zsv_cb.ext_sqlite3_add_csv(zdb, pd->data_filename, NULL, NULL) == SQLITE_OK) { - if (zsv_cb.ext_sheet_buffer_info(buff).has_row_num) - sqlite3_str_appendf(sql_str, "select *"); - else - sqlite3_str_appendf(sql_str, "select rowid as [Row #], *"); + sqlite3_str_appendf(sql_str, "select rowid as [Row #], *"); sqlite3_str_appendf(sql_str, " from data where %s = %Q", pd->value_sql, pr->value); - fprintf(stderr, "SQL: %s\n", sqlite3_str_value(sql_str)); + // fprintf(stderr, "SQL: %s\n", sqlite3_str_value(sql_str)); zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), NULL, NULL, NULL); } if (sql_str) @@ -327,7 +327,7 @@ enum zsv_ext_status zsv_ext_init(struct zsv_ext_callbacks *cb, zsv_execution_con int proc_id = zsv_cb.ext_sheet_register_proc("my-sheet-pivot", "my sheet pivot", my_pivot_table_command_handler); if (proc_id < 0) return zsv_ext_status_error; - zsv_cb.ext_sheet_register_proc_key_binding('v', proc_id); + zsv_cb.ext_sheet_register_proc_key_binding('s', proc_id); return zsv_ext_status_ok; } diff --git a/app/sheet.c b/app/sheet.c index a34ed39d..ab10f77c 100644 --- a/app/sheet.c +++ b/app/sheet.c @@ -49,8 +49,8 @@ struct zsvsheet_opts { #define ZSVSHEET_CELL_DISPLAY_MIN_WIDTH 10 static size_t zsvsheet_cell_display_width(struct zsvsheet_ui_buffer *ui_buffer, struct zsvsheet_display_dimensions *ddims) { - size_t width = ddims->columns / - (ui_buffer->dimensions.col_count + (ui_buffer->rownum_col_offset && !ui_buffer->has_row_num ? 1 : 0)); + size_t width = ddims->columns / (ui_buffer->dimensions.col_count + + (ui_buffer->rownum_col_offset && !ui_buffer->no_add_row_num ? 1 : 0)); return width < ZSVSHEET_CELL_DISPLAY_MIN_WIDTH ? ZSVSHEET_CELL_DISPLAY_MIN_WIDTH : width; } @@ -166,7 +166,8 @@ size_t display_data_rowcount(struct zsvsheet_display_dimensions *dims) { return dims->rows - dims->footer_span - dims->header_span; } -char zsvsheet_status_text[256] = {0}; +static int zsvsheet_status_priority; +static char zsvsheet_status_text[256]; static void zsvsheet_display_status_text(const struct zsvsheet_display_dimensions *ddims) { // clear the entire line mvprintw(ddims->rows - ddims->footer_span, 0, "%-*s", (int)sizeof(zsvsheet_status_text), ""); @@ -177,15 +178,24 @@ static void zsvsheet_display_status_text(const struct zsvsheet_display_dimension attroff(A_REVERSE); } -static void zsvsheet_priv_set_status(const struct zsvsheet_display_dimensions *ddims, int overwrite, const char *fmt, +#define ZSVSHEET_STATUS_LOW_PRIO 1 +#define ZSVSHEET_STATUS_HIGH_PRIO 10 + +static void zsvsheet_priv_set_status(const struct zsvsheet_display_dimensions *ddims, int priority, const char *fmt, ...) { - if (overwrite || !*zsvsheet_status_text) { + if (priority > zsvsheet_status_priority || !*zsvsheet_status_text) { va_list argv; va_start(argv, fmt); vsnprintf(zsvsheet_status_text, sizeof(zsvsheet_status_text), fmt, argv); va_end(argv); // note: if (n < (int)sizeof(zsvsheet_status_text)), then we just ignore + zsvsheet_status_priority = priority; } + + // The priority decays with each call which is at least n times per second as set by halfdelay + if (zsvsheet_status_priority > 0) + zsvsheet_status_priority--; + zsvsheet_display_status_text(ddims); } @@ -331,7 +341,7 @@ char zsvsheet_handle_find_next(struct zsvsheet_ui_buffer *uib, const char *needl *update_buffer = zsvsheet_goto_input_raw_row(uib, zsvsheet_opts->found_rownum, header_span, ddims, (size_t)-1); return 1; } - zsvsheet_priv_set_status(ddims, 1, "Not found"); + zsvsheet_priv_set_status(ddims, ZSVSHEET_STATUS_HIGH_PRIO, "Not found"); return 0; } @@ -394,11 +404,11 @@ static zsvsheet_status zsvsheet_open_file_handler(struct zsvsheet_proc_context * if ((err = zsvsheet_ui_buffer_open_file(filename, NULL, state->custom_prop_handler, di->ui_buffers.base, di->ui_buffers.current))) { if (err > 0) - zsvsheet_priv_set_status(di->dimensions, 1, "%s: %s", filename, strerror(err)); + zsvsheet_priv_set_status(di->dimensions, ZSVSHEET_STATUS_HIGH_PRIO, "%s: %s", filename, strerror(err)); else if (err < 0) - zsvsheet_priv_set_status(di->dimensions, 1, "Unexpected error"); + zsvsheet_priv_set_status(di->dimensions, ZSVSHEET_STATUS_HIGH_PRIO, "Unexpected error"); else - zsvsheet_priv_set_status(di->dimensions, 1, "Not found: %s", filename); + zsvsheet_priv_set_status(di->dimensions, ZSVSHEET_STATUS_HIGH_PRIO, "Not found: %s", filename); return zsvsheet_status_ignore; } no_input: @@ -462,16 +472,17 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx) struct zsvsheet_sheet_context *state = (struct zsvsheet_sheet_context *)ctx->subcommand_context; struct zsvsheet_display_info *di = &state->display_info; struct zsvsheet_screen_buffer_opts bopts = { - .no_rownum_column = 1, .cell_buff_len = 64, .max_cell_len = 0, .rows = 256, }; + struct zsvsheet_opts zsvsheet_opts = {0}; + zsvsheet_opts.hide_row_nums = 1; struct zsvsheet_ui_buffer_opts uibopts = { + .zsvsheet_opts = &zsvsheet_opts, .buff_opts = &bopts, .filename = NULL, .data_filename = NULL, - .no_rownum_col_offset = 1, .write_after_open = 0, }; struct zsvsheet_ui_buffer *uib = NULL; @@ -538,6 +549,7 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx) return stat; } +#include "sheet/pivot.c" #include "sheet/newline_handler.c" /* We do most procedures in one handler. More complex procedures can be @@ -624,7 +636,9 @@ struct builtin_proc_desc { { zsvsheet_builtin_proc_filter, "filter", "Hide rows that do not contain the specified text", zsvsheet_filter_handler }, { zsvsheet_builtin_proc_subcommand, "subcommand", "Editor subcommand", zsvsheet_subcommand_handler }, { zsvsheet_builtin_proc_help, "help", "Display a list of actions and key-bindings", zsvsheet_help_handler }, - { zsvsheet_builtin_proc_newline, "","Follow hyperlink (if any)", zsvsheet_newline_handler }, + { zsvsheet_builtin_proc_newline, "","Follow hyperlink (if any) or drill down", zsvsheet_newline_handler }, + { zsvsheet_builtin_proc_pivot_cur_col, "pivotcur","Group rows by the column under the cursor", zsvsheet_pivot_handler }, + { zsvsheet_builtin_proc_pivot_expr, "pivotexpr","Group rows with group-by SQL expression", zsvsheet_pivot_handler }, { -1, NULL, NULL, NULL } }; /* clang-format on */ @@ -642,7 +656,7 @@ static void zsvsheet_check_buffer_worker_updates(struct zsvsheet_ui_buffer *ub, struct zsvsheet_sheet_context *handler_state) { pthread_mutex_lock(&ub->mutex); if (ub->status) - zsvsheet_priv_set_status(display_dims, 1, ub->status); + zsvsheet_priv_set_status(display_dims, ZSVSHEET_STATUS_LOW_PRIO, ub->status); if (ub->index_ready && ub->dimensions.row_count != ub->index->row_count + 1) { ub->dimensions.row_count = ub->index->row_count + 1; handler_state->display_info.update_buffer = true; @@ -738,7 +752,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op ch = getch(); handler_state.display_info.update_buffer = false; - zsvsheet_priv_set_status(&display_dims, 1, ""); + zsvsheet_priv_set_status(&display_dims, ZSVSHEET_STATUS_LOW_PRIO, ""); if (ch != ERR) { status = zsvsheet_key_press(ch, &handler_state); @@ -753,9 +767,12 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op if (handler_state.display_info.update_buffer && zsvsheet_buffer_data_filename(ub)) { struct zsvsheet_opts zsvsheet_opts = {0}; - if (read_data(&ub, NULL, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col, header_span, - &zsvsheet_opts, custom_prop_handler)) { - zsvsheet_priv_set_status(&display_dims, 1, "Unexpected error!"); // to do: better error message + struct zsvsheet_ui_buffer_opts uibopts = {0}; + uibopts.zsvsheet_opts = &zsvsheet_opts; + if (read_data(&ub, &uibopts, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col, + header_span, custom_prop_handler)) { + zsvsheet_priv_set_status(&display_dims, ZSVSHEET_STATUS_HIGH_PRIO, + "Unexpected error!"); // to do: better error message continue; } } diff --git a/app/sheet/file.c b/app/sheet/file.c index 5e4122e9..81aceadd 100644 --- a/app/sheet/file.c +++ b/app/sheet/file.c @@ -1,3 +1,5 @@ +#include + int zsvsheet_ui_buffer_open_file_opts(struct zsvsheet_ui_buffer_opts *uibopts, struct zsv_prop_handler *custom_prop_handler, struct zsvsheet_ui_buffer **ui_buffer_stack_bottom, @@ -6,11 +8,14 @@ int zsvsheet_ui_buffer_open_file_opts(struct zsvsheet_ui_buffer_opts *uibopts, struct zsvsheet_screen_buffer_opts bopts = {0}; struct zsvsheet_ui_buffer *tmp_ui_buffer = NULL; + if (!uibopts->zsvsheet_opts) + uibopts->zsvsheet_opts = &zsvsheet_opts; + if (!uibopts->buff_opts) uibopts->buff_opts = &bopts; int err = 0; - if ((err = read_data(&tmp_ui_buffer, uibopts, 0, 0, 0, &zsvsheet_opts, custom_prop_handler)) != 0 || !tmp_ui_buffer || + if ((err = read_data(&tmp_ui_buffer, uibopts, 0, 0, 0, custom_prop_handler)) != 0 || !tmp_ui_buffer || !tmp_ui_buffer->buff_used_rows) { zsvsheet_ui_buffer_delete(tmp_ui_buffer); if (err) @@ -62,3 +67,17 @@ zsvsheet_status zsvsheet_open_file_opts(struct zsvsheet_proc_context *ctx, struc return zsvsheet_status_error; return zsvsheet_status_ok; } + +zsvsheet_status zsvsheet_ext_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_open_file_opts *opts) { + struct zsvsheet_ui_buffer_opts uibopts = {0}; + struct zsvsheet_opts zsvsheet_opts = {0}; + + zsvsheet_opts.hide_row_nums = opts->no_auto_row_num; + uibopts.zsvsheet_opts = &zsvsheet_opts; + uibopts.filename = opts->filename; + uibopts.data_filename = opts->data_filename; + if (opts->zsv_opts) + uibopts.zsv_opts = *opts->zsv_opts; + + return zsvsheet_open_file_opts(ctx, &uibopts); +} diff --git a/app/sheet/file.h b/app/sheet/file.h index 482db958..d1df2227 100644 --- a/app/sheet/file.h +++ b/app/sheet/file.h @@ -7,5 +7,6 @@ int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zs struct zsvsheet_ui_buffer **ui_buffer_stack_top); zsvsheet_status zsvsheet_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_ui_buffer_opts *opts); +zsvsheet_status zsvsheet_open_file(struct zsvsheet_proc_context *ctx, const char *filepath, struct zsv_opts *zopts); #endif diff --git a/app/sheet/handlers.c b/app/sheet/handlers.c index 4b6306c8..c3a3dd97 100644 --- a/app/sheet/handlers.c +++ b/app/sheet/handlers.c @@ -45,6 +45,7 @@ zsvsheet_status zsvsheet_set_status(struct zsvsheet_proc_context *ctx, const cha va_end(argv); // note: if (n < (int)sizeof(zsvsheet_status_text)), then we just ignore zsvsheet_display_status_text(state->display_info.dimensions); + zsvsheet_status_priority = ZSVSHEET_STATUS_HIGH_PRIO; return zsvsheet_status_ok; } @@ -112,7 +113,7 @@ zsvsheet_status zsvsheet_buffer_get_selected_cell(zsvsheet_buffer_t h, struct zs if (!uib) return zsvsheet_status_error; rc->row = uib->cursor_row + uib->input_offset.row + uib->buff_offset.row; - rc->col = uib->cursor_col + uib->input_offset.col + uib->buff_offset.row; + rc->col = uib->cursor_col + uib->input_offset.col + uib->buff_offset.col; return zsvsheet_status_ok; } @@ -191,7 +192,7 @@ struct zsvsheet_buffer_data zsvsheet_buffer_info(zsvsheet_buffer_t h) { struct zsvsheet_buffer_data d = {0}; struct zsvsheet_ui_buffer *b = h; if (b) { - d.has_row_num = b->has_row_num; + d.has_row_num = b->no_add_row_num; } return d; } diff --git a/app/sheet/handlers_internal.h b/app/sheet/handlers_internal.h index b639fe45..975007de 100644 --- a/app/sheet/handlers_internal.h +++ b/app/sheet/handlers_internal.h @@ -1,6 +1,8 @@ #ifndef ZSVSHEET_HANDLER_INTERNAL_H #define ZSVSHEET_HANDLER_INTERNAL_H +#include + struct zsvsheet_context { const char *subcommand_value; // e.g. "/path/to/myfile.csv" int ch; // key press value from getch() @@ -34,6 +36,11 @@ zsvsheet_status zsvsheet_open_file(struct zsvsheet_proc_context *ctx, const char /** extension support **/ +/** + * Open a tabular file with external facing options + */ +zsvsheet_status zsvsheet_ext_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_open_file_opts *opts); + /** * Set the subcommand prompt */ diff --git a/app/sheet/key-bindings.c b/app/sheet/key-bindings.c index c1b5bdc7..6eff5049 100644 --- a/app/sheet/key-bindings.c +++ b/app/sheet/key-bindings.c @@ -174,6 +174,8 @@ struct zsvsheet_key_binding zsvsheet_vim_key_bindings[] = { { .ch = '?', .proc_id = zsvsheet_builtin_proc_help, }, { .ch = '\n', .proc_id = zsvsheet_builtin_proc_newline, }, { .ch = '\r', .proc_id = zsvsheet_builtin_proc_newline, }, + { .ch = 'v', .proc_id = zsvsheet_builtin_proc_pivot_cur_col, }, + { .ch = 'V', .proc_id = zsvsheet_builtin_proc_pivot_expr, }, { .ch = -1 } }; diff --git a/app/sheet/pivot.c b/app/sheet/pivot.c new file mode 100644 index 00000000..2819884d --- /dev/null +++ b/app/sheet/pivot.c @@ -0,0 +1,317 @@ +/* + * Copyright (C) 2021 Liquidaty and zsv contributors. All rights reserved. + * This file is part of zsv/lib, distributed under the license defined at + * https://opensource.org/licenses/MIT + */ + +#include +#include +#include "../external/sqlite3/sqlite3.h" +#include +#include +#include +#include +#include +#include "file.h" +#include "handlers_internal.h" +#include "../sql_internal.h" + +struct pivot_row { + char *value; // to do: this will be the drill-down criteria +}; + +struct pivot_data { + char *value_sql; // the sql expression entered by the user e.g. City + char *data_filename; + struct zsv_opts zopts; + struct { + struct pivot_row *data; // for each row, the value of the sql expression e.g. New York + size_t capacity; + size_t used; + } rows; +}; + +static void pivot_data_delete(void *h) { + struct pivot_data *pd = h; + if (pd) { + for (size_t i = 0; i < pd->rows.used; i++) + free(pd->rows.data[i].value); + free(pd->rows.data); + free(pd->value_sql); + free(pd->data_filename); + free(pd); + } +} + +static struct pivot_data *pivot_data_new(const char *data_filename, const char *value_sql) { + struct pivot_data *pd = calloc(1, sizeof(*pd)); + if (pd && (pd->value_sql = strdup(value_sql)) && (pd->data_filename = strdup(data_filename))) + return pd; + pivot_data_delete(pd); + return NULL; +} + +#define ZSV_MYSHEET_PIVOT_DATA_ROWS_INITIAL 32 +static int pivot_data_grow(struct pivot_data *pd) { + if (pd->rows.used == pd->rows.capacity) { + size_t new_capacity = pd->rows.capacity == 0 ? ZSV_MYSHEET_PIVOT_DATA_ROWS_INITIAL : pd->rows.capacity * 2; + struct pivot_row *new_data = realloc(pd->rows.data, new_capacity * sizeof(*pd->rows.data)); + if (!new_data) + return ENOMEM; + pd->rows.data = new_data; + pd->rows.capacity = new_capacity; + } + return 0; +} + +static int add_pivot_row(struct pivot_data *pd, const char *value, size_t len) { + int err = pivot_data_grow(pd); + char *value_dup = NULL; + if (!err && value && len) { + value_dup = malloc(len + 1); + if (value_dup) { + memcpy(value_dup, value, len); + value_dup[len] = '\0'; + } + } + pd->rows.data[pd->rows.used++].value = value_dup; + return err; +} + +static struct pivot_row *get_pivot_row_data(struct pivot_data *pd, size_t row_ix) { + if (pd && row_ix < pd->rows.used) + return &pd->rows.data[row_ix]; + return NULL; +} + +// TO DO: return zsvsheet_status +static enum zsv_ext_status get_cell_attrs(void *pdh, zsvsheet_cell_attr_t *attrs, size_t start_row, size_t row_count, + size_t cols) { + struct pivot_data *pd = pdh; + size_t end_row = start_row + row_count; + int attr = 0; + +#ifdef A_BOLD + attr |= A_BOLD; +#endif +// Absent on Mac OSX 13 +#ifdef A_ITALIC + attr |= A_ITALIC; +#endif + + if (end_row > pd->rows.used) + end_row = pd->rows.used; + for (size_t i = start_row; i < end_row; i++) + attrs[i * cols] = attr; + return zsv_ext_status_ok; +} + +static void pivot_on_header_cell(void *ctx, size_t col_ix, const char *colname) { + (void)colname; + if (col_ix == 0) + add_pivot_row(ctx, NULL, 0); +} + +static void pivot_on_data_cell(void *ctx, size_t col_ix, const char *text, size_t len) { + if (col_ix == 0) + add_pivot_row(ctx, text, len); +} + +static zsvsheet_status zsv_sqlite3_to_csv(zsvsheet_proc_context_t pctx, struct zsv_sqlite3_db *zdb, const char *sql, + void *ctx, void (*on_header_cell)(void *, size_t, const char *), + void (*on_data_cell)(void *, size_t, const char *, size_t len)) { + const char *err_msg = NULL; + zsvsheet_status zst = zsvsheet_status_error; + sqlite3_stmt *stmt = NULL; + + if ((zdb->rc = sqlite3_prepare_v2(zdb->db, sql, -1, &stmt, NULL)) == SQLITE_OK) { + char *tmp_fn = zsv_get_temp_filename("zsv_mysheet_ext_XXXXXXXX"); + struct zsv_csv_writer_options writer_opts = zsv_writer_get_default_opts(); + zsv_csv_writer cw = NULL; + if (!tmp_fn) + zst = zsvsheet_status_memory; + else if (!(writer_opts.stream = fopen(tmp_fn, "wb"))) { + zst = zsvsheet_status_error; + err_msg = strerror(errno); + } else if (!(cw = zsv_writer_new(&writer_opts))) + zst = zsvsheet_status_memory; + else { + zst = zsvsheet_status_ok; + unsigned char cw_buff[1024]; + zsv_writer_set_temp_buff(cw, cw_buff, sizeof(cw_buff)); + + int col_count = sqlite3_column_count(stmt); + // write header row + for (int i = 0; i < col_count; i++) { + const char *colname = sqlite3_column_name(stmt, i); + zsv_writer_cell(cw, !i, (const unsigned char *)colname, colname ? strlen(colname) : 0, 1); + if (on_header_cell) + on_header_cell(ctx, i, colname); + } + + // write sql results + while (sqlite3_step(stmt) == SQLITE_ROW) { + for (int i = 0; i < col_count; i++) { + const unsigned char *text = sqlite3_column_text(stmt, i); + int len = text ? sqlite3_column_bytes(stmt, i) : 0; + zsv_writer_cell(cw, !i, text, len, 1); + if (on_data_cell) + on_data_cell(ctx, i, (const char *)text, len); + } + } + } + if (cw) + zsv_writer_delete(cw); + if (writer_opts.stream) + fclose(writer_opts.stream); + + if (tmp_fn && zsv_file_exists(tmp_fn)) { + struct zsvsheet_opts zsvsheet_opts = {0}; + zsvsheet_opts.hide_row_nums = 1; + struct zsvsheet_ui_buffer_opts uibopts = {0}; + uibopts.zsvsheet_opts = &zsvsheet_opts; + uibopts.data_filename = tmp_fn; + zst = zsvsheet_open_file_opts(pctx, &uibopts); + } else { + if (zst == zsvsheet_status_ok) { + zst = zsvsheet_status_error; // to do: make this more specific + if (!err_msg && zdb && zdb->rc != SQLITE_OK) + err_msg = sqlite3_errmsg(zdb->db); + } + } + if (zst != zsvsheet_status_ok) + free(tmp_fn); + } else { + err_msg = sqlite3_errmsg(zdb->db); + } + if (stmt) + sqlite3_finalize(stmt); + if (err_msg) + zsvsheet_set_status(pctx, "Error: %s", err_msg); + return zst; +} + +zsvsheet_status pivot_drill_down(zsvsheet_proc_context_t ctx) { + enum zsvsheet_status zst = zsvsheet_status_ok; + zsvsheet_buffer_t buff = zsvsheet_buffer_current(ctx); + struct pivot_data *pd; + struct zsvsheet_rowcol rc; + if (zsvsheet_buffer_get_ctx(buff, (void **)&pd) != zsv_ext_status_ok || + zsvsheet_buffer_get_selected_cell(buff, &rc) != zsvsheet_status_ok) { + return zsvsheet_status_error; + } + struct pivot_row *pr = get_pivot_row_data(pd, rc.row); + if (pd && pd->data_filename && pd->value_sql && pr) { + struct zsv_sqlite3_dbopts dbopts = {0}; + sqlite3_str *sql_str = NULL; + struct zsv_sqlite3_db *zdb = zsv_sqlite3_db_new(&dbopts); + + if (!zdb || !(sql_str = sqlite3_str_new(zdb->db))) + zst = zsvsheet_status_memory; + else if (zdb->rc == SQLITE_OK && zsv_sqlite3_add_csv(zdb, pd->data_filename, &pd->zopts, NULL) == SQLITE_OK) { + sqlite3_str_appendf(sql_str, "select rowid as [Row #], * from data where %s = %Q", pd->value_sql, pr->value); + zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), NULL, NULL, NULL); + } + + if (sql_str) + sqlite3_free(sqlite3_str_finish(sql_str)); + if (zdb) { + if (zst != zsvsheet_status_ok) { + // to do: consolidate this with same code in sql.c + if (zdb->err_msg) + fprintf(stderr, "Error: %s\n", zdb->err_msg); + else if (!zdb->db) + fprintf(stderr, "Error (unable to open db, code %i): %s\n", zdb->rc, sqlite3_errstr(zdb->rc)); + else if (zdb->rc != SQLITE_OK) + fprintf(stderr, "Error (code %i): %s\n", zdb->rc, sqlite3_errstr(zdb->rc)); + } + zsv_sqlite3_db_delete(zdb); + } + } + return zst; +} + +/** + * Here we define a custom command for the zsv `sheet` feature + */ +static zsvsheet_status zsvsheet_pivot_handler(struct zsvsheet_proc_context *ctx) { + char result_buffer[256] = {0}; + const char *expr; + size_t trim_len; + struct zsvsheet_rowcol rc; + int ch = zsvsheet_ext_keypress(ctx); + if (ch < 0) + return zsvsheet_status_error; + + zsvsheet_buffer_t buff = zsvsheet_buffer_current(ctx); + const char *data_filename = NULL; + if (buff) + data_filename = zsvsheet_buffer_data_filename(buff); + + if (!data_filename) { // TO DO: check that the underlying data is a tabular file and we know how to parse + zsvsheet_set_status(ctx, "Pivot table only available for tabular data buffers"); + return zsvsheet_status_ok; + } + + switch (ctx->proc_id) { + case zsvsheet_builtin_proc_pivot_expr: + zsvsheet_ext_prompt(ctx, result_buffer, sizeof(result_buffer), "Pivot table: Enter group-by SQL expr"); + if (*result_buffer == '\0') + return zsvsheet_status_ok; + expr = result_buffer; + break; + case zsvsheet_builtin_proc_pivot_cur_col: + if (zsvsheet_buffer_get_selected_cell(buff, &rc) != zsvsheet_status_ok) + return zsvsheet_status_error; + + expr = zsvsheet_ui_buffer_get_header(buff, rc.col); + assert(expr); + trim_len = strlen(expr); + expr = (char *)zsv_strtrim((unsigned char *)expr, &trim_len); + assert(expr); + assert(trim_len <= sizeof(result_buffer) - 1); + assert(trim_len > 0); + expr = sqlite3_snprintf(sizeof(result_buffer), result_buffer, "\"%.*w\"", trim_len, expr); + assert(strlen(expr) > trim_len); + break; + default: + assert(0); + return zsvsheet_status_error; + } + + assert(strlen(expr) > 0); + + enum zsvsheet_status zst = zsvsheet_status_ok; + struct zsv_sqlite3_dbopts dbopts = {0}; + struct zsv_opts zopts = zsvsheet_buffer_get_zsv_opts(buff); + struct zsv_sqlite3_db *zdb = zsv_sqlite3_db_new(&dbopts); + sqlite3_str *sql_str = NULL; + struct pivot_data *pd = NULL; + if (!zdb || !(sql_str = sqlite3_str_new(zdb->db))) + zst = zsvsheet_status_memory; + else if (zdb->rc == SQLITE_OK && zsv_sqlite3_add_csv(zdb, data_filename, &zopts, NULL) == SQLITE_OK) { + sqlite3_str_appendf(sql_str, "select %s as value, count(1) as Count from data group by %s", expr, expr); + if (!(pd = pivot_data_new(data_filename, expr))) + zst = zsvsheet_status_memory; + else { + zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), pd, pivot_on_header_cell, pivot_on_data_cell); + if (zst == zsvsheet_status_ok) { + buff = zsvsheet_buffer_current(ctx); + zsvsheet_buffer_set_ctx(buff, pd, pivot_data_delete); + zsvsheet_buffer_set_cell_attrs(buff, get_cell_attrs); + zsvsheet_buffer_on_newline(buff, pivot_drill_down); + pd->zopts = zopts; + pd = NULL; // so that it isn't cleaned up below + } + } + // TO DO: add param to ext_sheet_open_file to set filename vs data_filename, and set buffer type or proc owner + // TO DO: add way to attach custom context, and custom context destructor, to the new buffer + // TO DO: add cell highlighting + } + + zsv_sqlite3_db_delete(zdb); + if (sql_str) + sqlite3_free(sqlite3_str_finish(sql_str)); + pivot_data_delete(pd); + return zst; +} diff --git a/app/sheet/procedure.h b/app/sheet/procedure.h index 5c36a309..d81ac0c8 100644 --- a/app/sheet/procedure.h +++ b/app/sheet/procedure.h @@ -34,6 +34,8 @@ enum { zsvsheet_builtin_proc_help, zsvsheet_builtin_proc_vim_g_key_binding_dmux, zsvsheet_builtin_proc_newline, + zsvsheet_builtin_proc_pivot_expr, + zsvsheet_builtin_proc_pivot_cur_col, }; #define ZSVSHEET_PROC_INVALID 0 diff --git a/app/sheet/read-data.c b/app/sheet/read-data.c index f4878311..c8241d32 100644 --- a/app/sheet/read-data.c +++ b/app/sheet/read-data.c @@ -46,13 +46,13 @@ static void get_data_index_async(struct zsvsheet_ui_buffer *uibuffp, const char static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ui_buffer will be allocated struct zsvsheet_ui_buffer_opts *uibopts, // if *uibufferp == NULL and uibopts != NULL - size_t start_row, size_t start_col, size_t header_span, struct zsvsheet_opts *zsvsheet_opts, + size_t start_row, size_t start_col, size_t header_span, struct zsv_prop_handler *custom_prop_handler) { - const char *filename = (uibufferp && *uibufferp) ? (*uibufferp)->filename : uibopts ? uibopts->filename : NULL; + struct zsvsheet_opts *zsvsheet_opts = uibopts->zsvsheet_opts; struct zsv_opts opts = {0}; if (uibufferp && *uibufferp) opts = (*uibufferp)->zsv_opts; - else if (uibopts) + else opts = uibopts->zsv_opts; struct zsvsheet_ui_buffer *uibuff = uibufferp ? *uibufferp : NULL; @@ -60,6 +60,7 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ size_t remaining_header_to_skip = header_span; size_t original_row_num = 0; FILE *fp; + const char *filename = NULL; if (uibuff) { if (uibuff->data_filename) @@ -68,7 +69,7 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ filename = uibuff->filename; } - if (!filename && uibopts) { + if (!filename) { if (uibopts->data_filename) filename = uibopts->data_filename; else if (uibopts->filename) @@ -116,17 +117,17 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ size_t find_len = zsvsheet_opts->find ? strlen(zsvsheet_opts->find) : 0; size_t rows_searched = 0; zsvsheet_screen_buffer_t buffer = uibuff ? uibuff->buffer : NULL; - if (uibuff && uibuff->has_row_num) + if (uibuff && uibuff->no_add_row_num) zsvsheet_opts->hide_row_nums = 1; while (zsv_next_row(parser) == zsv_status_row && (rows_read == 0 || rows_read < zsvsheet_screen_buffer_rows(buffer))) { // for each row size_t col_count = zsv_cell_count(parser); - if (uibuff == NULL && uibufferp && uibopts && col_count > 0) { + if (uibuff == NULL && uibufferp && col_count > 0) { enum zsvsheet_priv_status stat; struct zsvsheet_ui_buffer *tmp_uibuff = NULL; - if (!(buffer = zsvsheet_screen_buffer_new(col_count, uibopts->buff_opts, &stat)) || + if (!(buffer = zsvsheet_screen_buffer_new(col_count + 1, uibopts->buff_opts, &stat)) || stat != zsvsheet_priv_status_ok || !(tmp_uibuff = zsvsheet_ui_buffer_new(buffer, uibopts))) { if (tmp_uibuff) zsvsheet_ui_buffer_delete(tmp_uibuff); @@ -138,16 +139,16 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ } // row number - size_t rownum_column_offset = 0; if (rows_read == 0 && zsvsheet_opts->hide_row_nums == 0) { // Check if we already have Row # struct zsv_cell c = zsv_get_cell(parser, 0); if (c.len == ZSVSHEET_ROWNUM_HEADER_LEN && !memcmp(c.str, ZSVSHEET_ROWNUM_HEADER, c.len)) { zsvsheet_opts->hide_row_nums = 1; if (uibuff) - uibuff->has_row_num = 1; + uibuff->no_add_row_num = 1; } } + size_t rownum_column_offset = !zsvsheet_opts->hide_row_nums; original_row_num++; if (remaining_header_to_skip > 0) { @@ -155,7 +156,7 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ continue; } if (uibuff) { - if (col_count + !buffer->opts.no_rownum_column > buffer->cols) { + if (col_count > buffer->cols) { if (zsvsheet_screen_buffer_grow(buffer, col_count) != zsvsheet_priv_status_ok) return -1; } @@ -187,7 +188,6 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ sprintf(buff, "########"); zsvsheet_screen_buffer_write_cell(buffer, rows_read, 0, (unsigned char *)buff); } - rownum_column_offset = 1; } for (size_t i = start_col; i < col_count && i + rownum_column_offset < zsvsheet_screen_buffer_cols(buffer); i++) { @@ -262,14 +262,16 @@ static void *get_data_index(void *gdi) { static size_t zsvsheet_find_next(struct zsvsheet_ui_buffer *uib, const char *needle, struct zsvsheet_opts *zsvsheet_opts, size_t header_span, struct zsv_prop_handler *custom_prop_handler) { + struct zsvsheet_ui_buffer_opts uibopts = {0}; struct zsvsheet_rowcol *input_offset = &uib->input_offset; struct zsvsheet_rowcol *buff_offset = &uib->buff_offset; size_t cursor_row = uib->cursor_row; zsvsheet_opts->find = needle; zsvsheet_opts->found_rownum = 0; + uibopts.zsvsheet_opts = zsvsheet_opts; // TO DO: check if it exists in current row, later column (and change 'cursor_row - 1' below to 'cursor_row') - read_data(&uib, NULL, input_offset->row + buff_offset->row + header_span + cursor_row - 1, 0, header_span, - zsvsheet_opts, custom_prop_handler); + read_data(&uib, &uibopts, input_offset->row + buff_offset->row + header_span + cursor_row - 1, 0, header_span, + custom_prop_handler); zsvsheet_opts->find = NULL; return zsvsheet_opts->found_rownum; } diff --git a/app/sheet/screen_buffer.c b/app/sheet/screen_buffer.c index 6008ed35..812adfc8 100644 --- a/app/sheet/screen_buffer.c +++ b/app/sheet/screen_buffer.c @@ -82,8 +82,6 @@ zsvsheet_screen_buffer_t zsvsheet_screen_buffer_new(size_t cols, struct zsvsheet if (opts->cell_buff_len < sizeof(void *) * 2) *stat = zsvsheet_priv_status_error; else { - if (!opts->no_rownum_column) - cols++; void *data = calloc(opts->rows, cols * opts->cell_buff_len); if (!data) *stat = zsvsheet_priv_status_memory; @@ -110,9 +108,6 @@ enum zsvsheet_priv_status zsvsheet_screen_buffer_grow(zsvsheet_screen_buffer_t b size_t cell_buff_len = buff->opts.cell_buff_len; size_t old_row_len = old_cols * cell_buff_len; - if (!buff->opts.no_rownum_column) - cols++; - size_t row_len = cols * cell_buff_len; assert(cols > old_cols); diff --git a/app/sheet/screen_buffer.h b/app/sheet/screen_buffer.h index cd1aa70a..a9012ed1 100644 --- a/app/sheet/screen_buffer.h +++ b/app/sheet/screen_buffer.h @@ -8,10 +8,9 @@ typedef struct zsvsheet_screen_buffer *zsvsheet_screen_buffer_t; struct zsvsheet_screen_buffer_opts { - size_t cell_buff_len; // default = 16. must be >= 2 * sizeof(void *) - size_t max_cell_len; // length in bytes; defaults to 32767 - size_t rows; // rows to buffer. cannot be < 256 - char no_rownum_column; // reserved. TO DO: if set, omit row num column + size_t cell_buff_len; // default = 16. must be >= 2 * sizeof(void *) + size_t max_cell_len; // length in bytes; defaults to 32767 + size_t rows; // rows to buffer. cannot be < 256 }; zsvsheet_screen_buffer_t zsvsheet_screen_buffer_new(size_t cols, struct zsvsheet_screen_buffer_opts *opts, diff --git a/app/sheet/transformation.c b/app/sheet/transformation.c index 5d58f8ff..2b423b38 100644 --- a/app/sheet/transformation.c +++ b/app/sheet/transformation.c @@ -269,8 +269,10 @@ enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx, goto error; } + struct zsvsheet_opts zsvsheet_opts = {0}; struct zsvsheet_ui_buffer_opts uibopts = {0}; + uibopts.zsvsheet_opts = &zsvsheet_opts; uibopts.data_filename = zsvsheet_transformation_filename(trn); uibopts.write_after_open = 1; diff --git a/app/sheet/ui_buffer.c b/app/sheet/ui_buffer.c index 8a4f4c8c..b2bb770c 100644 --- a/app/sheet/ui_buffer.c +++ b/app/sheet/ui_buffer.c @@ -40,7 +40,7 @@ struct zsvsheet_ui_buffer { unsigned char index_ready : 1; unsigned char rownum_col_offset : 1; unsigned char index_started : 1; - unsigned char has_row_num : 1; + unsigned char no_add_row_num : 1; unsigned char mutex_inited : 1; unsigned char write_in_progress : 1; unsigned char write_done : 1; @@ -92,11 +92,11 @@ void zsvsheet_ui_buffer_delete(struct zsvsheet_ui_buffer *ub) { } struct zsvsheet_ui_buffer_opts { + struct zsvsheet_opts *zsvsheet_opts; struct zsvsheet_screen_buffer_opts *buff_opts; const char *filename; const char *data_filename; struct zsv_opts zsv_opts; // options to use when opening this file - char no_rownum_col_offset; char write_after_open; }; @@ -108,9 +108,14 @@ struct zsvsheet_ui_buffer *zsvsheet_ui_buffer_new(zsvsheet_screen_buffer_t buffe uib->buffer = buffer; uib->mutex_inited = 1; memcpy(&uib->mutex, &init, sizeof(init)); - if (!(uibopts && uibopts->no_rownum_col_offset)) - uib->rownum_col_offset = 1; + uib->rownum_col_offset = !uibopts; + if (uibopts) { + if (uibopts->zsvsheet_opts->hide_row_nums) + uib->no_add_row_num = 1; + else + uib->rownum_col_offset = 1; + if (uibopts->filename && !(uib->filename = strdup(uibopts->filename))) { zsvsheet_ui_buffer_delete(uib); return NULL; @@ -192,3 +197,9 @@ int zsvsheet_ui_buffer_pop(struct zsvsheet_ui_buffer **base, struct zsvsheet_ui_ } return 0; } + +static const char *zsvsheet_ui_buffer_get_header(struct zsvsheet_ui_buffer *uib, size_t col) { + struct zsvsheet_screen_buffer *sb = uib->buffer; + + return (char *)zsvsheet_screen_buffer_cell_display(sb, 0, col); +} diff --git a/app/sql.c b/app/sql.c index 816758bd..02ced918 100644 --- a/app/sql.c +++ b/app/sql.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include "sql_internal.h" #include // unlink @@ -109,8 +111,6 @@ static char is_select_sql(const char *s) { (const unsigned char *)s, strlen("select ")); } -#include "sql_internal.c" - int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *opts, struct zsv_prop_handler *custom_prop_handler) { /** diff --git a/app/test/Makefile b/app/test/Makefile index cf56d255..99ea1d2d 100644 --- a/app/test/Makefile +++ b/app/test/Makefile @@ -618,7 +618,7 @@ test-sheet-cleanup: @rm -f tmux-*.log @tmux kill-server || printf '' -test-sheet-all: test-sheet-1 test-sheet-2 test-sheet-3 test-sheet-4 test-sheet-5 test-sheet-6 test-sheet-7 test-sheet-8 test-sheet-9 test-sheet-10 test-sheet-11 test-sheet-12 test-sheet-13 test-sheet-14 test-sheet-subcommand test-sheet-prop-cmd-opt +test-sheet-all: test-sheet-1 test-sheet-2 test-sheet-3 test-sheet-4 test-sheet-5 test-sheet-6 test-sheet-7 test-sheet-8 test-sheet-9 test-sheet-10 test-sheet-11 test-sheet-12 test-sheet-13 test-sheet-14 test-sheet-subcommand test-sheet-prop-cmd-opt test-sheet-pivot @(for SESSION in $^; do ! tmux kill-session -t "$$SESSION" 2>/dev/null; done && ${TEST_PASS} || ${TEST_FAIL}) TMUX_TERM=xterm-256color @@ -807,6 +807,43 @@ test-sheet-prop-cmd-opt: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${BUILD_DIR}/bin/zsv_p # @tmux send-keys -t $@ "q" # @${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}) +test-sheet-pivot: test-sheet-pivot-1 test-sheet-pivot-2 test-sheet-pivot-3 + +test-sheet-pivot-1: ${BUILD_DIR}/bin/zsv_sheet${EXE} + @${TEST_INIT} + @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf + @(tmux new-session -x 80 -y 25 -d -s "$@" "${PREFIX} $< worldcitiespop_mil.csv" && \ + ${EXPECT} $@ indexed && \ + tmux send-keys -t $@ l v && \ + ${EXPECT} $@ groups && \ + tmux send-keys -t $@ j j Enter && \ + ${EXPECT} $@ drilldown && \ + tmux send-keys -t $@ G && \ + ${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL}) + +test-sheet-pivot-2: ${BUILD_DIR}/bin/zsv_sheet${EXE} + @${TEST_INIT} + @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf + @(tmux new-session -x 80 -y 25 -d -s "$@" "${PREFIX} $< -d 3 ../../data/test/mixed-line-endings.csv" && \ + ${EXPECT} $@ indexed && \ + tmux send-keys -t $@ l l l v && \ + ${EXPECT} $@ groups && \ + tmux send-keys -t $@ j j Enter && \ + ${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL}) + +test-sheet-pivot-3: ${BUILD_DIR}/bin/zsv_sheet${EXE} + @${TEST_INIT} + @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf + @(tmux new-session -x 80 -y 25 -d -s "$@" "${PREFIX} $< worldcitiespop_mil.csv" && \ + ${EXPECT} $@ indexed && \ + tmux send-keys -t $@ V && \ + ${EXPECT} $@ prompt && \ + tmux send-keys -l -t $@ "iif(coalesce(Population, '') = '', 'Unknown', 'Populated')" && \ + tmux send-keys -t $@ Enter && \ + ${EXPECT} $@ groups && \ + tmux send-keys -t $@ Enter && \ + ${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL}) + benchmark-sheet-index: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV} @${TEST_INIT} @if [ "${BIG_FILE}" = "none" ]; then \ diff --git a/app/test/expected/test-sheet-pivot-1-drilldown.out b/app/test/expected/test-sheet-pivot-1-drilldown.out new file mode 100644 index 00000000..bd14fadf --- /dev/null +++ b/app/test/expected/test-sheet-pivot-1-drilldown.out @@ -0,0 +1,25 @@ +Row # Country City AccentCit Region Populatio Latitude Longitude +546 af abakeh Abakeh 24 36.67363 69.078194 +549 af abakheyl' Abakheyl' 35 34.744167 70.287778 +551 af aband Aband 42 35.474953 69.871144 +556 af abasali Abasali 39 32.813402 65.968042 +558 af abaskhank Abaskhank 29 32.503976 69.209795 +560 af abas khel Abas Khel 28 31.975853 67.59182 +561 af abaskheyl Abaskheyl 28 31.975853 67.59182 +563 af `abaskust `Abaskust 27 34.331309 67.640744 +576 af ab barik Ab Barik 09 34.651024 66.254311 +577 af `abbas `Abbas 08 33.333 67.643278 +582 af `abbaskha `Abbaskha 29 32.469231 69.20712 +583 af `abbas kh `Abbas Kh 29 32.469231 69.20712 +588 af `abbas kh `Abbas Kh 29 32.671879 69.134674 +589 af `abbaskhe `Abbaskhe 29 32.671879 69.134674 +590 af abbaskhel Abbaskhel 29 32.831993 69.12336 +593 af `abbaskhe `Abbaskhe 29 32.831389 69.260556 +594 af abbaskhey Abbaskhey 29 32.671879 69.134674 +596 af `abbaskhe `Abbaskhe 29 32.831993 69.12336 +600 af `abbas ku `Abbas Ku 27 34.331309 67.640744 +604 af abbazha-y Abbazha-y 10 31.791857 64.566416 +607 af ab bazha- Ab Bazha- 10 31.719049 64.600709 +608 af abbazha-y Abbazha-y 10 31.719049 64.600709 +611 af abbedak Abbedak 09 33.482286 64.310608 +? for help 546 diff --git a/app/test/expected/test-sheet-pivot-1-groups.out b/app/test/expected/test-sheet-pivot-1-groups.out new file mode 100644 index 00000000..05b370a3 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-1-groups.out @@ -0,0 +1,25 @@ +value Count +ad 27 +ae 147 +af 27822 +ag 55 +ai 14 +al 4730 +am 896 +an 75 +ao 6170 +ar 2817 +at 4740 +au 3403 +aw 37 +az 3522 +ba 5076 +bb 165 +bd 8280 +be 5056 +bf 3248 +bg 6258 +bh 102 +bi 646 +bj 1331 +? for help ad diff --git a/app/test/expected/test-sheet-pivot-1-indexed.out b/app/test/expected/test-sheet-pivot-1-indexed.out new file mode 100644 index 00000000..610fb180 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-1-indexed.out @@ -0,0 +1,25 @@ +Row # Country City AccentCit Region Populatio Latitude Longitude +1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207 +2 ad aixirival Aixirival 06 42.466666 1.5 +3 mm mokho-atw Mokho-atw 09 18.033333 96.75 +4 id selingon Selingon 17 -8.8374 116.4914 +5 ir berimvand Berimvand 13 34.2953 47.1096 +6 pl chomiaza Chomiaza 73 52.7508 17.841793 +7 mg itona Itona 02 -23.86666 47.216666 +8 ad andorra-v Andorra-V 07 42.5 1.5166667 +9 mx la tortug La Tortug 25 25.75 -108.3333 +10 us asaph Asaph PA 41.770833 -77.40527 +11 ad andorre-v Andorre-V 07 42.5 1.5166667 +12 ru dolmatova Dolmatova 71 57.436791 63.279522 +13 ro escu Escu 13 47.133333 23.533333 +14 us la presa La Presa CA 35119 32.708055 -116.9963 +15 pk makam khu Makam Khu 04 33.650863 72.551536 +16 ad aubinya Aubinyà 06 42.45 1.5 +17 mm kiosong Kiosöng 11 22.583333 97.05 +18 tr donencay Dönençay 28 40.266667 38.583333 +19 it roncaglia Roncaglia 05 45.05 9.8 +20 ml kourmouss Kourmouss 04 14.75 -5.033333 +21 id lamogo Lamogo 38 -4.3945 119.9028 +22 ad casas vil Casas Vil 03 42.533333 1.5666667 +23 ru otdeleniy Otdeleniy 86 51.726473 39.714345 +? for help 1 diff --git a/app/test/expected/test-sheet-pivot-1.out b/app/test/expected/test-sheet-pivot-1.out new file mode 100644 index 00000000..d2f20474 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-1.out @@ -0,0 +1,25 @@ +Row # Country City AccentCit Region Populatio Latitude Longitude +89222 af zungur ka Zungur Ka 27 33.993056 68.658056 +89223 af zunun Zunun 01 38.438492 70.904387 +89224 af zu ol faq Zu ol Faq 03 35.616782 68.670307 +89225 af zu ol faq Zu ol Faq 27 34.1575 68.501389 +89232 af zur baraw Zur Baraw 34 35.028791 71.507898 +89235 af zuri Zuri 11 33.94062 62.160912 +89238 af zur kalay Zur Kalay 35 34.670345 70.165192 +89244 af zur kowt Zur Kowt 37 33.542222 69.734167 +89246 af zurmatiya Zurmatiya 28 32.138117 66.835273 +89248 af zurmat Zurmat 36 33.437782 69.02774 +89249 af zurni Zurni 09 33.568987 63.289317 +89252 af zurwam Zurwam 23 31.439167 66.936667 +89256 af zutni khe Zutni Khe 36 33.360477 68.973862 +89258 af zutni khe Zutni Khe 36 33.360477 68.973862 +89261 af zu Zu 01 36.84154 71.069323 +89262 af zvaka Zvaka 29 32.968334 68.809853 +89264 af zwaka Zwaka 29 32.968334 68.809853 +89267 af zyan say Zyan Say 07 35.864952 64.278152 +89279 af zyaratjay Zyaratjay 11 34.197386 62.140143 +89281 af zyarat ka Zyarat Ka 18 34.288611 71.138889 +89282 af zyarat ka Zyarat Ka 23 31.355826 66.535299 +89283 af zyarat ka Zyarat Ka 35 34.738611 70.269444 +89286 af zyarat Zyarat 39 32.523283 65.905489 +? for help 89286 diff --git a/app/test/expected/test-sheet-pivot-2-groups.out b/app/test/expected/test-sheet-pivot-2-groups.out new file mode 100644 index 00000000..d933c39e --- /dev/null +++ b/app/test/expected/test-sheet-pivot-2-groups.out @@ -0,0 +1,25 @@ +value Count + C1 1 + C10 1 + C100 1 + C1000 1 + C1001 1 + C1002 1 + C1003 1 + C1004 1 + C1005 1 + C1006 1 + C1007 1 + C1008 1 + C1009 1 + C101 1 + C1010 1 + C1011 1 + C1012 1 + C1013 1 + C1014 1 + C1015 1 + C1016 1 + C1017 1 + C1018 1 +? for help C1 diff --git a/app/test/expected/test-sheet-pivot-2-indexed.out b/app/test/expected/test-sheet-pivot-2-indexed.out new file mode 100644 index 00000000..b7f86738 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-2-indexed.out @@ -0,0 +1,25 @@ +Row # HA1 HA2 HA3 HB1 HB2 HB3 HC1 HC2 HC3 +1 A1 B1 C1 +2 A2 B2 C2 +3 A3 B3 C3 +4 A4 B4 C4 +5 A5 B5 C5 +6 A6 B6 C6 +7 A7 B7 C7 +8 A8 B8 C8 +9 A9 B9 C9 +10 A10 B10 C10 +11 A11 B11 C11 +12 A12 B12 C12 +13 A13 B13 C13 +14 A14 B14 C14 +15 A15 B15 C15 +16 A16 B16 C16 +17 A17 B17 C17 +18 A18 B18 C18 +19 A19 B19 C19 +20 A20 B20 C20 +21 A21 B21 C21 +22 A22 B22 C22 +23 A23 B23 C23 +? for help 1 diff --git a/app/test/expected/test-sheet-pivot-2.out b/app/test/expected/test-sheet-pivot-2.out new file mode 100644 index 00000000..17285ddd --- /dev/null +++ b/app/test/expected/test-sheet-pivot-2.out @@ -0,0 +1,25 @@ +Row # HA1 HA2 HA3 HB1 HB2 HB3 HC1 HC2 HC3 +100 A100 B100 C100 + + + + + + + + + + + + + + + + + + + + + + +? for help 100 diff --git a/app/test/expected/test-sheet-pivot-3-groups.out b/app/test/expected/test-sheet-pivot-3-groups.out new file mode 100644 index 00000000..98a9d692 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-3-groups.out @@ -0,0 +1,25 @@ +value Count +Populated 15166 +Unknown 984834 + + + + + + + + + + + + + + + + + + + + + +? for help Populated diff --git a/app/test/expected/test-sheet-pivot-3-indexed.out b/app/test/expected/test-sheet-pivot-3-indexed.out new file mode 100644 index 00000000..610fb180 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-3-indexed.out @@ -0,0 +1,25 @@ +Row # Country City AccentCit Region Populatio Latitude Longitude +1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207 +2 ad aixirival Aixirival 06 42.466666 1.5 +3 mm mokho-atw Mokho-atw 09 18.033333 96.75 +4 id selingon Selingon 17 -8.8374 116.4914 +5 ir berimvand Berimvand 13 34.2953 47.1096 +6 pl chomiaza Chomiaza 73 52.7508 17.841793 +7 mg itona Itona 02 -23.86666 47.216666 +8 ad andorra-v Andorra-V 07 42.5 1.5166667 +9 mx la tortug La Tortug 25 25.75 -108.3333 +10 us asaph Asaph PA 41.770833 -77.40527 +11 ad andorre-v Andorre-V 07 42.5 1.5166667 +12 ru dolmatova Dolmatova 71 57.436791 63.279522 +13 ro escu Escu 13 47.133333 23.533333 +14 us la presa La Presa CA 35119 32.708055 -116.9963 +15 pk makam khu Makam Khu 04 33.650863 72.551536 +16 ad aubinya Aubinyà 06 42.45 1.5 +17 mm kiosong Kiosöng 11 22.583333 97.05 +18 tr donencay Dönençay 28 40.266667 38.583333 +19 it roncaglia Roncaglia 05 45.05 9.8 +20 ml kourmouss Kourmouss 04 14.75 -5.033333 +21 id lamogo Lamogo 38 -4.3945 119.9028 +22 ad casas vil Casas Vil 03 42.533333 1.5666667 +23 ru otdeleniy Otdeleniy 86 51.726473 39.714345 +? for help 1 diff --git a/app/test/expected/test-sheet-pivot-3-prompt.out b/app/test/expected/test-sheet-pivot-3-prompt.out new file mode 100644 index 00000000..edaa6c71 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-3-prompt.out @@ -0,0 +1,25 @@ +Row # Country City AccentCit Region Populatio Latitude Longitude +1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207 +2 ad aixirival Aixirival 06 42.466666 1.5 +3 mm mokho-atw Mokho-atw 09 18.033333 96.75 +4 id selingon Selingon 17 -8.8374 116.4914 +5 ir berimvand Berimvand 13 34.2953 47.1096 +6 pl chomiaza Chomiaza 73 52.7508 17.841793 +7 mg itona Itona 02 -23.86666 47.216666 +8 ad andorra-v Andorra-V 07 42.5 1.5166667 +9 mx la tortug La Tortug 25 25.75 -108.3333 +10 us asaph Asaph PA 41.770833 -77.40527 +11 ad andorre-v Andorre-V 07 42.5 1.5166667 +12 ru dolmatova Dolmatova 71 57.436791 63.279522 +13 ro escu Escu 13 47.133333 23.533333 +14 us la presa La Presa CA 35119 32.708055 -116.9963 +15 pk makam khu Makam Khu 04 33.650863 72.551536 +16 ad aubinya Aubinyà 06 42.45 1.5 +17 mm kiosong Kiosöng 11 22.583333 97.05 +18 tr donencay Dönençay 28 40.266667 38.583333 +19 it roncaglia Roncaglia 05 45.05 9.8 +20 ml kourmouss Kourmouss 04 14.75 -5.033333 +21 id lamogo Lamogo 38 -4.3945 119.9028 +22 ad casas vil Casas Vil 03 42.533333 1.5666667 +23 ru otdeleniy Otdeleniy 86 51.726473 39.714345 +Pivot table: Enter group-by SQL expr: diff --git a/app/test/expected/test-sheet-pivot-3.out b/app/test/expected/test-sheet-pivot-3.out new file mode 100644 index 00000000..d4f0d5f8 --- /dev/null +++ b/app/test/expected/test-sheet-pivot-3.out @@ -0,0 +1,25 @@ +Row # Country City AccentCit Region Populatio Latitude Longitude +14 us la presa La Presa CA 35119 32.708055 -116.9963 +54 ad les escal Les Escal 08 15854 42.5 1.5333333 +72 us lawrence Lawrence MA 72609 42.706944 -71.16361 +94 ae abu dhabi Abu Dhabi 01 603687 24.466667 54.366667 +208 ph general l General L H3 3132 10.833333 123.56666 +225 gr mitikas Mítikas 31 934 38.666666 20.933333 +307 ru shudayag Shudayag 34 3446 63.526652 53.605822 +385 ru shcheglov Shcheglov 42 3095 60.001399 30.783441 +573 sk prievidza Prievidza 01 52987 48.766666 18.633333 +640 jp matsubara Matsubara 32 130857 34.566667 135.55 +660 gr velventos Velventó 11 3378 40.251666 22.063055 +679 pt arazede Arazede 07 5906 40.286268 -8.649989 +716 ru porkhov Porkhov 60 11886 57.765016 29.556121 +751 us winter sp Winter Sp FL 32658 28.698611 -81.30833 +824 ru kubinka Kubinka 47 25942 55.579569 36.703919 +830 no alsvag Alsvåg 09 335 68.9 15.283333 +913 pl jozefow Jozefow 78 17910 51.806452 21.220953 +914 us del rio Del Rio TX 36068 29.362500 -100.8963 +1190 ru kamenka Kamenka 06 3258 65.887591 44.112612 +1330 lt venta Venta 40 3270 56.2 22.7 +1432 ro farcasu Farcasu 26 3620 44.6 23.75 +1532 sl largo Largo 03 4484 8.2666667 -12.15 +1562 pk eminabad Eminabad 04 22677 32.042188 74.259435 +? for help 14 diff --git a/include/zsv/ext.h b/include/zsv/ext.h index 392399a2..44ca3a91 100644 --- a/include/zsv/ext.h +++ b/include/zsv/ext.h @@ -238,6 +238,11 @@ struct zsv_ext_callbacks { */ zsvsheet_status (*ext_sheet_open_file)(zsvsheet_proc_context_t, const char *filepath, struct zsv_opts *zopts); + /** + * Open a tabular file with more available options + */ + zsvsheet_status (*ext_sheet_open_file_opts)(struct zsvsheet_proc_context *ctx, struct zsvsheet_open_file_opts *opts); + /** * Set custom context * @param on_close optional callback to invoke when the buffer is closed diff --git a/include/zsv/ext/sheet.h b/include/zsv/ext/sheet.h index f3e6b7e6..51c1e1c0 100644 --- a/include/zsv/ext/sheet.h +++ b/include/zsv/ext/sheet.h @@ -1,6 +1,8 @@ #ifndef ZSVSHEET_H #define ZSVSHEET_H +#include + /* custom sheet handler id */ typedef int zsvsheet_proc_id_t; @@ -22,6 +24,13 @@ typedef struct zsvsheet_subcommand_context *zsvsheet_subcommand_context_t; typedef void *zsvsheet_buffer_t; // int zsvsheet_ext_keypress(zsvsheet_proc_context_t); +struct zsvsheet_open_file_opts { + const char *filename; + const char *data_filename; + struct zsv_opts *zsv_opts; + char no_auto_row_num; +}; + typedef struct zsvsheet_transformation *zsvsheet_transformation; /** * Transformation options passed to zsvsheet_push_transformation