diff --git a/Makefile.am b/Makefile.am index f93057acc..e69973152 100644 --- a/Makefile.am +++ b/Makefile.am @@ -24,6 +24,7 @@ AM_DISTCHECK_CONFIGURE_FLAGS = --enable-debug \ --disable-root-actions \ --with-noaaport \ --with-retrans \ + --with-ingester \ --with-gribinsert \ LDMHOME=`cd ../_inst && pwd` # @DISTCHECK_CONFIGURE_FLAGS_WITH_MULTICAST@ @@ -65,6 +66,9 @@ SUBDIRS += lib if WANT_NOAAPORT SUBDIRS += noaaport endif +if WANT_INGESTER + SUBDIRS += fileIngest +endif SUBDIRS += regutil \ feedme \ hupsyslog \ @@ -196,6 +200,10 @@ install_setuids: chown root $(DESTDIR)$(bindir)/noaaportIngester; \ chmod 4755 $(DESTDIR)$(bindir)/noaaportIngester; \ fi + -if test -f $(DESTDIR)$(bindir)/fileIngester; then \ + chown root $(DESTDIR)$(bindir)/fileIngester; \ + chmod 4755 $(DESTDIR)$(bindir)/fileIngester; \ + fi -if test -f $(DESTDIR)$(bindir)/dvbs_multicast; then \ chown root $(DESTDIR)$(bindir)/dvbs_multicast; \ chmod 4755 $(DESTDIR)$(bindir)/dvbs_multicast; \ @@ -211,8 +219,8 @@ install_setuids: echo " \ ERROR: One or more of the following programs in the \"bin/\" installation \ directory is not owned by \"root\" or does not have the setuid bit enabled: \ -ldmd, hupsyslog, noaaportIngester, dvbs_multicast. The superuser will have to \ -set these attributes manually."; \ +ldmd, hupsyslog, noaaportIngester, fileIngester, dvbs_multicast. The superuser \ +will have to set these attributes manually."; \ echo; \ exit 1; \ fi diff --git a/configure.ac b/configure.ac index d045abf9d..97e3e2537 100644 --- a/configure.ac +++ b/configure.ac @@ -219,6 +219,14 @@ AC_ARG_WITH([noaaport], [with_noaaport=no]) AM_CONDITIONAL([WANT_NOAAPORT], [test "x$with_noaaport" != xno]) +AC_ARG_WITH([ingester], + AS_HELP_STRING([--with-ingester], + [enable support for ingesting files into product queue + [default=without]]), + [], + [with_ingester=no]) +AM_CONDITIONAL([WANT_INGESTER], [test "x$with_ingester" != xno]) + AC_ARG_WITH([retrans], AS_HELP_STRING([--with-retrans], [enable support for retransmission from a NOAAPORT receiver @@ -625,6 +633,15 @@ AM_COND_IF([WANT_NOAAPORT], [ [AC_MSG_ERROR([Library not found])]) ]) +AM_COND_IF([WANT_INGESTER],[ + AC_SEARCH_LIBS([clock_gettime], [rt],,[AC_MSG_ERROR([Could not find required function clock_gettime],[1])]) + AC_SEARCH_LIBS([timer_create], [rt],,[AC_MSG_ERROR([Could not find required function timer_create],[1])]) + AC_SEARCH_LIBS([timer_settime], [rt],,[AC_MSG_ERROR([Could not find required function timer_settime],[1])]) +libs=$LIBS +]) + + + # Ensure references to the XML2 package, which is used by the LDM registry # module. AC_CHECK_HEADER([libxml/parser.h], , @@ -665,6 +682,7 @@ AC_PROG_LIBTOOL AC_CONFIG_FILES([ fauxPq/Makefile feedme/Makefile + fileIngest/Makefile gempak/Makefile gempak/tables/Makefile grib2/Makefile diff --git a/fileIngest/.gitignore b/fileIngest/.gitignore new file mode 100644 index 000000000..2474a325b --- /dev/null +++ b/fileIngest/.gitignore @@ -0,0 +1,10 @@ +/test +/fileIngester +/test.c +/log_tester +/Makefile.in +/stdclib.h +/mlogger.h +/datelib.h +/goesr_lib.h + diff --git a/fileIngest/Makefile.am b/fileIngest/Makefile.am new file mode 100644 index 000000000..cd43a8f26 --- /dev/null +++ b/fileIngest/Makefile.am @@ -0,0 +1,141 @@ +# Copyright 2011 University Corporation for Atmospheric Research +# +# This file is part of the Unidata LDM package. See the file COPYRIGHT in the +# top-level source-directory of the package for copying and redistribution +# conditions. +# +## Process this file with automake(1) to produce file Makefile.in + +################################################################################ + +LDMHOME = @LDMHOME@ +LDMSRC = $(top_srcdir) +SUDO = @SUDO@ +SU = @SU@ + +CLEANFILES = $(BUILT_SOURCES) + +CPPFLAGS = \ + -DLDM_SUPPORT \ + -I$(top_srcdir) \ + -I$(top_srcdir)/noaaport \ + -I$(top_srcdir)/protocol \ + -I$(top_srcdir)/protocol2 \ + -I$(top_srcdir)/pq \ + -I$(top_srcdir)/ulog \ + -I$(top_srcdir)/log \ + -I$(top_srcdir)/misc \ + -I$(top_srcdir)/registry \ + @CPPFLAGS@ + +bin_PROGRAMS = fileIngester + +EXTRA_DIST = \ + datelib.hin \ + stdclib.hin \ + goesr_lib.hin \ + mlogger.hin + +BUILT_SOURCES = \ + datelib.h \ + stdclib.h \ + goesr_lib.h \ + mlogger.h + +COMMON_SOURCES = \ + stdclib.c stdclib.h \ + goesr_lib.c goesr_lib.h \ + mlogger.c mlogger.h \ + datelib.c datelib.h \ + ldmProductQueue.c ldmProductQueue.h + +fileIngester_SOURCES = fileIngester.c $(COMMON_SOURCES) + +LDADD = $(top_builddir)/lib/libldm.la + +TAGS_FILES = \ + *.c *.h + +.c.i: + $(COMPILE) -E $< >$@ + +.hin.h: + ./extractDecls $*.hin $*.c >$@.tmp + mv -f $@.tmp $@ + +#installcheck-local: +# pqcreate -c -s 2m /tmp/readnoaaport-test.pq +# $(DESTDIR)$(bindir)/readnoaaport -l- -q /tmp/readnoaaport-test.pq \ +# $(srcdir)/nwstgdump.data +# rm /tmp/readnoaaport-test.pq + +#check-local: check-readnoaaport +#check-readnoaaport: readnoaaport +# pqcreate -c -s 2m /tmp/readnoaaport-test.pq +# ./readnoaaport -nl- -q /tmp/readnoaaport-test.pq \ +# $(srcdir)/nwstgdump.data +# rm /tmp/readnoaaport-test.pq + +#root-ni-ck: +# ./noaaportIngester -b 3 -q /tmp/noaaportIngester-test.pq \ +# <$(srcdir)/nwstgdump.data + +#valgrind: valgrind-readnoaaport + +#valgrind-readnoaaport: readnoaaport +# pqcreate -c -s 2m /tmp/readnoaaport-test.pq +# $(LIBTOOL) --mode=execute valgrind --leak-check=yes \ +# readnoaaport -l- -q /tmp/readnoaaport-test.pq nwstgdump.data +# rm /tmp/readnoaaport-test.pq + +sudo: +if HAVE_SUDO + $(SUDO) $(MAKE) $(AM_MAKEFLAGS) $(TARGET) 2>/dev/tty +else +if HAVE_SU + @printf "Enter root's password (or don't): " >/dev/tty + @$(SU) root -c 'PATH='$$PATH' $(MAKE) $(AM_MAKEFLAGS) $(TARGET)' \ + /dev/tty + @echo >/dev/tty +endif +endif + +#valgrind-noaaportIngester: noaaportIngester +# pqcreate -c -s 2m /tmp/noaaportIngester-test.pq +# $(MAKE) $(AM_MAKEFLAGS) sudo TARGET=root-ni-vg +# rm /tmp/noaaportIngester-test.pq + +#root-ni-vg: +# $(LIBTOOL) --mode=execute valgrind --leak-check=yes \ +# noaaportIngester -n -q /tmp/noaaportIngester-test.pq \ +# <$(srcdir)/nwstgdump.data + +#debug-readnoaaport: readnoaaport +# pqcreate -c -s 2m /tmp/readnoaaport-test.pq +# echo 'handle SIGCONT pass noprint nostop' >/tmp/readnoaaport.gdb +# echo 'b 617' >>/tmp/readnoaaport.gdb +# echo 'run -l- -q /tmp/readnoaaport-test.pq nwstgdump.data' \ +# >>/tmp/readnoaaport.gdb +# $(LIBTOOL) --mode=execute gdb -x /tmp/readnoaaport.gdb readnoaaport +# rm /tmp/readnoaaport-test.pq /tmp/readnoaaport.gdb + +#debug-noaaportIngester: noaaportIngester +# pqcreate -c -s 2m /tmp/noaaportIngester-test.pq +# echo 'handle SIGCONT pass noprint nostop' >/tmp/noaaportIngester.gdb +# echo 'handle SIGTERM pass nostop' >>/tmp/noaaportIngester.gdb +# echo 'run -q /tmp/noaaportIngester-test.pq -n -m 224.0.1.1' \ +# >>/tmp/noaaportIngester.gdb +# $(MAKE) $(AM_MAKEFLAGS) sudo TARGET=root-ni-db +# rm /tmp/noaaportIngester-test.pq /tmp/noaaportIngester.gdb + +#root-ni-db: +# $(LIBTOOL) --mode=execute gdb -x /tmp/noaaportIngester.gdb \ +# noaaportIngester + +.PHONY: \ + sudo +# install-html \ +# root-ni-ck \ +# root-ni-db \ +# root-ni-vg \ +# sudo diff --git a/fileIngest/Makefile.local b/fileIngest/Makefile.local new file mode 100644 index 000000000..0e9d8b922 --- /dev/null +++ b/fileIngest/Makefile.local @@ -0,0 +1,38 @@ + +COPTS = -g -Wall + +BUILT_HEADERS = datelib.h goesr_lib.h mlogger.h stdclib.h +BUILT_PROGS = fileIngester test read_acq_pipe log_tester +INGEST_OBJS = fileIngester.o stdclib.o goesr_lib.o mlogger.o + +all :: fileIngester test log_tester read_acq_pipe + +clean :: + rm -f $(INGEST_OBJS) $(BUILT_PROGS) read_acq_pipe.o log_tester.o + +%.h : %.hin %.c extractDecls + ./extractDecls $*.hin $*.c > $@ + +%.o : %.c + cc $(COPTS) -o $@ -c $< + +goesr_lib.o : stdclib.h mlogger.h goesr_lib.h +stdclib.o : stdclib.h +fileIngester.o : stdclib.h mlogger.h goesr_lib.h +log_tester.o : stdclib.h mlogger.h + +fileIngester : $(INGEST_OBJS) + cc -g -lrt -o $@ $^ + +read_acq_pipe : read_acq_pipe.o stdclib.o + cc -g -lrt -o $@ $^ + +acq_ftp_send : $(INGEST_OBJS) + cc -g -lrt -o $@ $^ + +test : test.o + cc -g -o $@ $^ + +log_tester : log_tester.o mlogger.o stdclib.o + cc -g -lrt -o $@ $^ + diff --git a/fileIngest/datelib.c b/fileIngest/datelib.c new file mode 100644 index 000000000..462451cdf --- /dev/null +++ b/fileIngest/datelib.c @@ -0,0 +1,186 @@ +/* ------------------------------------------------------------------------------ + * + * File Name: + * datelib.c + * + * Description: + * This file contains functions for manipulating dates and times. + * + * Functions defined: + * format_date_string + * + * Author: + * Brian Rapp Jun 19, 2012 + * + * Modification History: + * Modified by Date + * Description + * + ------------------------------------------------------------------------------ */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include + +#include "datelib.h" + +/* ------------------------------------------------------------------------------ + +Function Name + formatDateString +Synopsis + Creates custom-formatted date strings. + +Prototype + int formatDateString (char *in_format, char *out_format, long adjustment, + char *in_date, char *out_date) + +Arguments + + Name: in_format + Type: char * + Access: read only + + Character string containing the format specification for the input time. + See the man page for strftime for details. This is only used when an + absolute input date is provided. + + Name: out_format + Type: char * + Access: read only + + Character string containing the Format specification for the output time. + See the man page for strftime for details. This argument is required. + + Name: adjustment + Type: long + Access: read only + + A long integer containing an adjustment (in seconds) to be applied to + the input time. Negative values adjust the time prior to the input + time. Positive values adjust the time after the input time. A value of + zero indicates no adjustment is to be made to the input time. + + Name: in_date + Type: char * + Access: read only + + Character string containing the input time. Can be an absolute time in + any format supported by the strftime function, or one of the following + relative times: + + NOW Current date and time + TODAY Today at midnight + YESTERDAY Yesterday at midnight + TOMORROW Tomorrow at midnight + + Name: out_date + Type: char * + Access: write only + + A character string that is large enough to hold the output date provided + in the out_format argument. The string must be allocated by the caller. + + Name: max_out_size + Type: int + Access: read only + + An integer containing the maximum size for out_date. This value must + include the trailing nul character. + +Description + This function creates a formatted date string optionally adjusted by + a specified amount. The output format defaults to "YYYY-MM-DD HH:MM:SS", + but can be fully specified with the 'out_format' argument. See the + man page for strftime for more information on formatting. + + An input time string is required, which can either be an absolute time + of the form "YYYY-MM-DD HH:MM:SS" or one of the relative time strings + "NOW", "YESTERDAY", "TODAY", or "TOMORROW". + + Another optional parameter provides the + ability to adjust the time string by the specified number of seconds. + +Return Values + The number of characters placed in out_date, not including the terminating + nul terminator, provided the string, including the terminating nul byte, + fits. Otherwise, it returns 0, and the contents of the array is undefined. + +------------------------------------------------------------------------------ */ + +int formatDateString (const char *in_format, const char *out_format, long adjustment, + char *in_date, char *out_date, int max_out_size) { + + struct tm *tp; + struct tm ts; + time_t the_time; + char *res; + size_t stat; + time_t time_adjust = 0; + char new_out_format[MAX_DATE_FORMAT_LEN+1]; + + if ((out_format == NULL) || (strlen (out_format) == 0)) { + strcpy (new_out_format, DEFAULT_OUT_FORMAT); + } else { + strncpy (new_out_format, out_format, MAX_DATE_FORMAT_LEN); + } + + if (!strcmp (in_date, RELT_NOW)) { + time (&the_time); + } else { + if (!strcmp (in_date, RELT_TODAY)) { + time (&the_time); + tp = localtime (&the_time); + tp->tm_hour = tp->tm_min = tp->tm_sec = 0; + the_time = mktime (tp); + } else { + if (!strcmp (in_date, RELT_YESTERDAY)) { + time (&the_time); + tp = localtime (&the_time); + tp->tm_hour = tp->tm_min = tp->tm_sec = 0; + the_time = mktime (tp); + the_time -= 86400; + } else { + if (!strcmp (in_date, RELT_TOMORROW)) { + time (&the_time); + tp = localtime (&the_time); + tp->tm_hour = tp->tm_min = tp->tm_sec = 0; + the_time = mktime (tp); + the_time += 86400; + } else { + memset (&ts, 0, sizeof (struct tm)); + res = strptime (in_date, in_format, &ts); /* Convert time string to broken-down time -- dst may be wrong */ + if ((res == 0) || (*res != 0)) { + return 0; + } + + /* Convert broken-down time back to calendar time so we have a proper value for tm_isdst */ + the_time = mktime (&ts); /* Convert broken-down time to calendar time */ + localtime_r (&the_time, &ts); /* Convert calendar time back to broken-down time -- dst will be set properly here */ + + res = strptime (in_date, in_format, &ts); + + if ((ts.tm_year == 0) || (ts.tm_mday == 0)) { + return 0; + } + the_time = mktime (&ts); + } + } + } + } + + the_time += time_adjust; + memset (&ts, 0, sizeof (struct tm)); + tp = localtime_r (&the_time, &ts); /* This will set tm_isdst properly for the given time */ + stat = strftime (out_date, max_out_size, new_out_format, &ts); + + return stat; + +} diff --git a/fileIngest/datelib.hin b/fileIngest/datelib.hin new file mode 100644 index 000000000..186f78b6a --- /dev/null +++ b/fileIngest/datelib.hin @@ -0,0 +1,32 @@ +/* ------------------------------------------------------------------------------ + * + * File Name: + * datelib.h + * + * Description: + * + * + * + * Author: + * brapp Jun 27, 2012 + * + * Modification History: + * Modified by Date + * Description + * +------------------------------------------------------------------------------ */ + + +#ifndef DATELIB_H_ +#define DATELIB_H_ + +#define RELT_NOW "NOW" +#define RELT_TODAY "TODAY" +#define RELT_YESTERDAY "YESTERDAY" +#define RELT_TOMORROW "TOMORROW" +#define MAX_DATE_FORMAT_LEN 128 +#define DEFAULT_OUT_FORMAT "%Y-%m-%d %H:%M:%S" + +@FUNCTION_DECLARATIONS@ + +#endif /* DATELIB_H_ */ diff --git a/fileIngest/extractDecls b/fileIngest/extractDecls new file mode 100755 index 000000000..2a2053bb3 --- /dev/null +++ b/fileIngest/extractDecls @@ -0,0 +1,38 @@ +templateFile=${1:?'header template-file not specified'} +cFile=${2:?'C source-file not specified'} +sed '/@FUNCTION_DECLARATIONS@/,$d' $templateFile \ +&& awk ' + BEGIN { + lineSep = ""; + } + /^\/\* -/ { + declStarted = 1; + lineCount = 0; + } + + /^static/ { + lineCount = 0; + declStarted = 0; + } + + /[[:print:]]*\{$/ { + if (declStarted) { + lines[++lineCount] = substr($0, 1, length($0)-1); + sub(/[ \t]+$/, "", lines[lineCount]); + for (i = 1; i <= lineCount; i++) { + printf ("%s%s", lineSep, lines[i]); + lineSep = "\n"; + } + print ";"; + } + declStarted = 0; + } + + /^struct/ || /^typedef/ || /^}/ { + declStarted = 0; + } + + declStarted != 0 { + lines[++lineCount] = $0; + }' $cFile \ +&& sed '1,/@FUNCTION_DECLARATIONS@/d' $templateFile \ No newline at end of file diff --git a/fileIngest/fileIngester.c b/fileIngest/fileIngester.c new file mode 100644 index 000000000..6b48e0c31 --- /dev/null +++ b/fileIngest/fileIngester.c @@ -0,0 +1,2958 @@ +/* ----------------------------------------------------------------------------- + * + * File Name: + * fileIngester.c + * + * Description: + * This program initiates processing on weather product files using one of several + * possible input methods and performing one of several output actions. It runs as + * either a standalone daemon or as a subprocess of LDM to insert files into a + * product queue. As a standalone daemon, it can either discard files, or write them + * to another directory as a front-end to uplink_send. This latter purpose is now + * largely obsolete since the same input methods are now supported natively by + * uplink_send. + * + * On input, fileIngester can either poll a single input directory, or read files + * to process from a named pipe (from uplink_send through acqserver). Polling method + * can be: GOESR, NDE, PDA, or POLL. + * + * The GOESR method relies on each file being renamed to a specific extension when + * it's ready to be processed. Full products are broken into smaller tiles to be able + * to meet program latency requirements. For each full product, a Product Activity + * Report (XML file) is transmitted after all related tiles are sent. The PAR must + * be processed by a separate external program. To support PAR processing, + * fileIngester can create individual files in a separate directory containing a hash + * code of each tile. GOES-R product files do not contain the required WMO + * header, which this program will automatically generate from the file name. If + * the output is to an LDM queue, the WMO header will be added automatically. If + * not, then the '-w' option must be provided for the WMO header to be added to + * the output file. + * + * NDE and PDA both use a 'marker' file to indicate a matching product file is + * ready for ingest. The NDE and PDA methods are identical: the marker file for each + * product will contain a hash code for the product file that fileIngester can process + * directly and output non-matching results to the error log. Regardless of status, + * each file will be ingested. + * + * The POLL method just assumes files are ready whenever they can be opened. + * + * To output to an LDM product queue, fileIngester must be specifically built as + * an LDM extension (see build instructions below). + * + * Dependencies: + * mlogger, ldmProductQueue, stdclib, goesr_lib + * + * Build Instructions: + * By default, LDM support is not compiled in. To use LDM, the source must + * be compiled as part of LDM using the Makefile.am automake template. This + * is part of a branch of v6.13.0 of LDM that includes changes to the LDM + * build system. To build it for LDM, follow the normal procedure. When + * ready to build, do the following from the base LDM source directory as + * user ldm: + * + * autoreconf -fi + * automake (if version errors occur, follow instructions provided) + * ./configure --with-ingester --prefix=$HOME + * rm -f libtool + * ln -s /usr/bin/libtool + * make + * make install + * + * Configuration + * ------------- + * + * This program can be used with LDM in several ways, or it can be used without + * LDM at all: + * + * 1. On a standalone server as upstream LDM feeding a downstream LDM + * + * In addition to the standalone upstream LDM server that this software will + * be installed on, the downstream LDM instance must be configured to pull + * RaFTR-generated products. If NIS is used, ensure the RaFTR host and the + * machine hosting the downstream LDM are defined. If not using NIS, define + * both in the local /etc/hosts file on each machine. After LDM has been + * successfully built and installed, configure as follows: + * + * RaFTR Server + * + * Ensure portmapper/rpcbind is running. + * + * In /etc/services, add or modify to obtain the following: + * + * ldm 388/tcp ldm + * ldm 388/udp ldm + * + * In /etc/rpc, add the following if not already present: + * + * ldm 300029 ldmd + * + * In /usr/local/ldm/etc/ldmd.conf, add an entry to allow the downstream + * LDM instance to access it. It should be something like this (with TAB + * characters between each field: + * + * ALLOW ANY ^.*$ + * + * where is a regular expression, such as: cpsbn.-tbwo + * + * Remove or comment out any lines that begin with "EXEC" and add one like: + * + * EXEC "fileIngester -p -L -M \ + * -iGOESR -oLDM -q -a0 -d2 -n2" + * + * Update the LDM registry (/usr/local/ldm/etc/registry.xml) with reasonable + * values. Be sure to updated the clock and provide a fully-qualified + * name for the local host. + * + * A sample system service file (ldm-raftr) in in /usr/local/ldm/src/ldm/raftr + * if needed. + * + * + * Downstream LDM Server + * + * In /usr/local/ldm/etc/ldmd.conf, add an entry to request data from raftr + * (white space must be tabs). For instance, if the upstream LDM instance + * were raftr-tbwo, this would be used: + * + * REQUEST ANY ".*" raftr-tbwo + * + * In /usr/local/ldm/etc/pqact.conf, add an entry for the GOES-R WMO headers + * using tabs as field separators. For instance: + * + * ANY ^(TI[RS]...) (KNES) (..)(..)(..) (...) + * FILE -overwrite -log -close -edex /data_store/goesr/\1_\2_\3\4\5_\6.nc + * + * Start LDM on both the upstream and downstream servers. Check ldmd.log on + * the downstream servers for issues connecting with the upstream servers. + * + * 2. On the machine hosting RaFTR feeding a downstream LDM + * + * The only difference between this scenario and the first one is that RaFTR + * will not be copying products across the network. Instead, RaFTR will be + * writing products locally. Configuration is exactly the same as above, but + * RaFTR must be configured to use some file transfer protocol (such as + * SFTP, SCP, or FTPS) to copy the files to this machine. + * + * 3. On a CPSBN that is also receiving an SBN stream via noaaportIngester + * + * 4. Without LDM to generate SBN-ready GOES-R products with WMO headers + * + * 5. Without LDM to write PDA and NDE products to directories for ingest with + * uplink_send. + * + * 6. To exercise and validate RaFTR without additional downstream processing. + * + * 7. To ingest products into a product queue through uplink_send/acqserver. + * + * To compile without LDM support, use the provided Makefile.local file: + * + * make -f Makefile.local fileIngester + * + * Functions defined: + * + * Author: + * Brian M Rapp 17-Nov-2013 + * + * Modification History: + * Modified by Date + * Description + * + * Brian Rapp Wed Dec 3 15:43:24 2014 -0500 + * Added NDE/PDA processing to read_raftr + * + * Brian Rapp Thu Feb 5 15:44:17 2015 -0500 + * Made exit summary message always log + * + * Brian Rapp Thu Feb 5 15:54:15 2015 -0500 + * Modified summary statistics to use product log instead of error log + * + * Brian M. Rapp Wed Apr 1 08:52:50 2015 -0400 + * Support for new types/cats/codes + * + * Brian Rapp Thu Apr 23 15:41:34 2015 -0400 + * Gave read_raftr.c the ability to generate checksum files for each + * tile file it processes. + * + * Brian Rapp Sun May 17 13:54:34 2015 -0400 + * Refactored read_raftr to fileIngester + * + * Brian Rapp Sat May 23 15:57:51 2015 -0400 + * Added NDE/PDA processing to fileIngester.c. + * + * Brian Rapp Mon May 25 10:59:17 2015 -0400 + * Changed directory creation permissions in fileIngester.c + * + * commit 51b37cd3ac498413a363812a2c224c282e3be364 + * Brian Rapp Fri Jun 5 15:36:06 2015 -0400 + * Add -h option to fileIngester + * + * commit cb55f0d8dbb3156ef2af69155840f921c2091bc8 + * Brian Rapp Tue Jun 9 15:15:48 2015 -0400 + * Fixed bug preventing product files from being deleted for + * OutType==LDM in fileIngester.c + * + * commit 25b2f76e3988b50e50b9e99d71d194b6a589438a + * Brian Rapp Thu Jun 18 16:18:31 2015 -0400 + * Fixed fileIngester bug that caused a 2nd WMO header to be + * added for PDA products written to an LDM queue. + * + * Brian Rapp 25-Jun-2015 + * Added support for the POLL input method. + * + * Brian Rapp 26-Jun-2015 + * Added support for the ACQ_PIPE input method. + * + * -------------------------------------------------------------------------- */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef LDM_SUPPORT +# include +# include +# include "ldmProductQueue.h" +#endif + +#include "../fileIngest/stdclib.h" +#include "../fileIngest/mlogger.h" +#include "../fileIngest/goesr_lib.h" + +#define DIRECTORY_CREATE_PERMS (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) +#define DIRECTORY_FULL_OPEN_PERMS (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IWOTH | S_IXOTH) +#define OUTFILE_FINAL_PERMS (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) +#define OUTFILE_CREATE_PERMS S_IWUSR +#define DEFAULT_VERBOSITY V_ERROR +#define DEFAULT_POLLING_INTERVAL 2 /* Default directory polling interval in seconds */ +#define SLEEP_TIME_SECS 1 /* Time to sleep between loop iterations */ +#define DEFAULT_MAX_QUEUE_SIZE 500 +#define DEFAULT_FILE_SPEC "*" +#define DEFAULT_MAX_SAVE_FILES 500 +#define DEFAULT_SAVE_FILE_DIGITS 3 +#define MAX_FILENAME_LEN 128 +#define MAX_PATH_LEN 256 +#define MAX_ACQ_PATH_LEN 128 +#define MAX_HOST_LEN 64 +#define MAX_HASH_LEN 128 +#define MIN_DISCARD_AGE 60 /* Minimum age for DiscardAge in seconds */ +#define DEFAULT_DISCARD_AGE 3600 /* Default discard age in seconds */ +#define PROD_LOG "products.log" +#define PROD_LOG_PATH "/awips/logs/Products" +#define MESSAGE_LOG "messages.log" +#define MESSAGE_LOG_PATH "/awips/logs/Messages" +#define DEF_LDM_QUEUE "/awips/ldm/data/ldm.pq" +#define DEF_OUTFILE_PREFIX "goesr" +#define DEF_LOG_SIZE (4 * 1024 * 1024) +#define TRACE_LOG_SIZE (100 * 1024 * 1024) +#define LOG_BUFFER_SIZE 1024 +#define MIN_PROD_SIZE_READ 25 +#define STATUS_FREQUENCY 50 +#define MAX_HOST_NAME_LEN 64 /* max length for host name */ +#define DEF_STR_LEN 128 + +#define SIZE_WMO_HDR 24 +#define SIZE_WMO_TERM 3 +#define WMO_TERMINATOR "\r\r\n" +#define SIZE_SBN_HDR 11 +#define SIZE_SBN_TLR 4 +#define SBN_HEADER_TEMPLATE "\001\015\015\012%03d\040\015\015\012" +#define SBN_TRAILER "\015\015\012\003" +#define DEFAULT_FEED_TYPE OTHER + +#define OUT_NONE 0 +#define OUT_DISCARD 1 +#define OUT_FILE 2 +#ifdef LDM_SUPPORT +# define OUT_LDM 4 +#endif + +#define IN_NONE 0 +#define IN_GOESR 1 +#define IN_NDE 2 +#define IN_PDA 3 +#define IN_POLL 4 +#define IN_ACQ_PIPE 5 + +#define PROD_TYPE_NWSTG 5 +#define PROD_CAT_NWSTG 101 +#define SBN_TYPE_ID_GOESR 12 +#define PROD_CAT_IMAGE 3 +#define PC_WMO_SAT_IMAGE_T 52 + +#define STAT_ERROR -1 +#define STAT_SUCCESS 0 +#define STAT_MORE_FILES 1 +#define STAT_ALREADY_QUEUED 3 + +#define MIN(a, b) ((a < b) ? a : b) + +typedef struct pipe_prod_name_hdr { /* prod filename hdr for pipe */ + int pipe_insert_time; /* time product added to pipe */ + int pipe_prod_NCF_rcv_time; /* time product received at NCF */ + ushort pipe_prod_type; /* product category */ + /* TYPE_GOES, TYPE_NWSTG, etc */ + ushort pipe_prod_cat; /* product category */ + /* CAT_IMAGE, etc */ + ushort pipe_prod_code; /* product code 1,2,3,etc */ + ushort pipe_prod_flag; /* product flag (error & status) */ + /* error_mask & prod_done,etc */ + ushort reserve1; /* reserved */ + ushort reserve2; /* reserved */ + uint pipe_prod_orig_prod_seqno; /* original prod seqno to retransmit */ + int pipe_prod_orig_NCF_rcv_time; /* NCF receive time binary(GMT) */ + /* Retransmit info */ + ushort pipe_prod_run_id; /* Unique run identification */ + /* for product stream */ + /* parm for retransmission */ + ushort pipe_prod_orig_run_id; /* Unique orig run identification */ + char pipe_prod_filename[MAX_ACQ_PATH_LEN]; /* UNIX filename for prod */ +} PIPE_PROD_NAME_HDR; + +typedef struct file_node { + char *fptr; /* File name (without path) */ + time_t mtime; /* File last modification time */ + off_t fsize; /* File size in bytes */ +} FILE_NODE; + +typedef struct file_list_hdr { + FILE_NODE *fileNodes; /* Pointer to array of FILE_NODEs */ + int count; /* Number of files in list */ +} FILE_LIST; + +typedef struct optdef { + char *str; + unsigned int val; +} OPT_SPEC; + +OPT_SPEC inOpts[] = { { "GOESR", IN_GOESR }, + { "NDE", IN_NDE }, + { "PDA", IN_PDA }, + { "POLL", IN_POLL }, + { "ACQ_PIPE", IN_ACQ_PIPE }, + { NULL, IN_NONE } + }; + +OPT_SPEC outOpts[] = { { "DISCARD", OUT_DISCARD }, + { "FILE", OUT_FILE }, +#ifdef LDM_SUPPORT + { "LDM", OUT_LDM }, +#endif + { NULL, OUT_NONE } + }; + +#define MD5 1 +#define SHA1 2 +#define SHA224 3 +#define SHA256 4 +#define SHA384 5 +#define SHA512 6 + +OPT_SPEC csOpts[] = { { "MD5", MD5 }, + { "SHA1", SHA1 }, + { "SHA224", SHA224 }, + { "SHA256", SHA256 }, + { "SHA384", SHA384 }, + { "SHA512", SHA512 }, + { NULL, 0 } + }; + +char *hashProgs[] = { "md5sum", + "sha1sum", + "sha224sum", + "sha256sum", + "sha384sum", + "sha512sum" + }; + +#ifdef LDM_SUPPORT +OPT_SPEC feedOpts[] = { { "IMAGE", IMAGE }, + { "TEXT", TEXT }, + { "GRID", GRID }, + { "POINT", POINT }, + { "BUFR", BUFR }, + { "GRAPH", GRAPH }, + { "OTHER", OTHER }, + { "NEXRAD", NEXRAD }, + { "NPORT", NPORT } + }; +#endif + +extern char *optarg; + +pid_t MyPid; +char *ProgName; +char LocalHostName[MAX_HOST_NAME_LEN+1]; +int PollInterval; /* Default time to sleep if no files in polling directory */ +int SleepPollInterval; /* Time to sleep between poll iterations */ +int Done; /* Set when need to shut down */ +int SaveFiles; /* If set, processed files are saved to the 'sent' directory */ +int MaxSentFiles; /* If using -d, maximum # of files to write before starting over */ +int SentFileDigits; /* Number of digits to output in sent file names */ +int SaveFails; /* If set, files that failed valiation are saved to 'fail' directory */ +int OutAction; /* Output options: OUT_DISCARD, OUT_FILE, OUT_LDM, OUT_ACQSERVER */ +int InType; /* Input options: IN_GOESR, IN_NDE */ +int Validate; /* Validate input files or not. Validation method depends on InType */ +int CreateChecksum; /* Create checksum of input file contents and write to .hash file */ +int HashOpt; +char *HashProgram; /* Program executable name */ +int MaxQueueSize; /* Maximum number of products to read from the polling directory per cycle */ +int AddLdmWrapper; /* If set, an LDM wrapper (header and trailer) is added to each product */ +int AddWmoHeader; /* If set, a WMO header is calculated and added to each product */ +char InputSource[MAX_PATH_LEN+1]; /* Directory to poll for products or pipe name with -iPIPE */ +char Loc[5]; /* Input location string - either "pipe" or "dir" */ +char FailDir[MAX_PATH_LEN+1]; /* Directory to write failed products */ +char SentDir[MAX_PATH_LEN+1]; /* Directory to write sent products */ +char SaveDir[MAX_PATH_LEN+1]; /* Directory to save products to when -o FILE option is provided along with -d */ +char ParDir[MAX_PATH_LEN+1]; /* Directory to save PAR files to (GOES-R) */ +char HostName[MAX_HOST_LEN+1]; /* Local host name */ +char PollFileSpec[MAX_FILENAME_LEN+1]; /* File specification for products that are ready to ingest, can include wildcards */ +char LogPathBase[MAX_PATH_LEN+1]; /* Product log directory */ +char MessagePath[MAX_PATH_LEN+1]; /* Error/Debug log directory */ +int DiscardAge; /* Files older than this (in seconds) will be discarded */ +int Verbosity; /* Verbosity level of error log */ +LOGGER *pLog; /* Product logger */ +LOGGER *eLog; /* Error logger */ +char *ProdBuf; /* Buffer used to read products from disk. Reallocated as necessary */ +int ProdBufSize; /* Current size of prodBuf */ +unsigned long long TotalProductsProcessed; /* Total number of products processed since start up */ +unsigned long long TotalBytesProcessed; /* Total number of bytes processed since start up */ +int SbnSeqNo; /* Product sequence number for LDM */ +int SentSeqNo; /* Sequence number when the -of option is provided */ + +#ifdef LDM_SUPPORT + +feedtypet FeedType; /* LDM Feed Type -- only valid with -oLDM */ +char pqfName[MAX_PATH_LEN+1]; /* LDM product queue file path */ +LdmProductQueue *prodQueue; /* LDM queue structure */ +MD5_CTX *md5ctxp; /* MD5 context for generating hash codes used by LDM */ + +#endif + +/* ----------------------------------------------------------------------------- + * Function Name + * usage + * + * Format + * static void usage (const char *progname) + * + * Arguments + * char *progname + * Character string containing the name of this program. + * + * Description + * Print program usage information and exit. + * + * Return Values + * This function does not return, it exits with value 1 (error). + * + * -------------------------------------------------------------------------- */ + +static void usage (const char *progname) { + + fprintf (stderr, "usage: %s -p -i (ACQ_PIPE|GOESR|NDE|PDA|POLL)\n" +#ifdef LDM_SUPPORT + "\t-o (DISCARD|FILE|LDM) [-q |-d ] [-F ] [-w]\n" +#else + "\t-o (DISCARD|FILE) [-d ] [-w]\n" +#endif + "\t[-c ] [-h ] [-x ]\n" + "\t[-s ] [-f ] [-n ]\n" + "\t[-t