Skip to content

Commit

Permalink
Merge pull request #1142 from US-CBP/GTASv1.1
Browse files Browse the repository at this point in the history 
Gtasv1.1
  • Loading branch information
@originalname51
originalname51 authored May 29, 2019
2 parents f2aca60 + 1aef863 commit 9946425
Show file tree
Hide file tree
Showing 149 changed files with 16,691 additions and 20,684 deletions.
Binary file modified gtas-neo4j-etl/doc/GTAS-Neo4j-deployment-instruction.docx
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create-flight-destination.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -841,7 +841,7 @@
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create-flight-origin.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -837,7 +837,7 @@
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create-passenger-address.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -868,7 +868,7 @@
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create-passenger-creditcard.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -868,7 +868,7 @@
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create-passenger-email.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,7 @@
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create-passenger-phone.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -868,7 +868,7 @@
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
2 changes: 1 addition & 1 deletion gtas-neo4j-etl/job/gtas-create_passenger-document.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -882,7 +882,7 @@ All GTAS code is Copyright 2016, The Department of Homeland Security (DHS), U.S.
<trim_type>both</trim_type>
</field>
<field>
<name>citizenship_country</name>
<name>nationality</name>
<type>String</type>
<format/>
<currency>$</currency>
Expand Down
61 changes: 46 additions & 15 deletions gtas-neo4j-etl/job/gtas-passenger-hits.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,15 +480,15 @@
</attribute>
<attribute>
<code>INITIAL_POOL_SIZE</code>
<attribute>5</attribute>
<attribute>6</attribute>
</attribute>
<attribute>
<code>IS_CLUSTERED</code>
<attribute>N</attribute>
</attribute>
<attribute>
<code>MAXIMUM_POOL_SIZE</code>
<attribute>8</attribute>
<attribute>10</attribute>
</attribute>
<attribute>
<code>PORT_NUMBER</code>
Expand Down Expand Up @@ -573,12 +573,7 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Add constants</from>
<to>Select values</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Select values</from>
<from>Select values - format dates-ids</from>
<to>Set field value to a constant</to>
<enabled>Y</enabled>
</hop>
Expand All @@ -592,6 +587,16 @@
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add constants</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Select values - format dates-ids</to>
<enabled>Y</enabled>
</hop>
</order>
<step>
<name>Abort</name>
Expand All @@ -618,7 +623,7 @@
</remotesteps>
<GUI>
<xloc>480</xloc>
<yloc>784</yloc>
<yloc>752</yloc>
<draw>Y</draw>
</GUI>
</step>
Expand Down Expand Up @@ -705,7 +710,7 @@
</remotesteps>
<GUI>
<xloc>112</xloc>
<yloc>912</yloc>
<yloc>1008</yloc>
<draw>Y</draw>
</GUI>
</step>
Expand Down Expand Up @@ -833,11 +838,37 @@
</output>
</remotesteps>
<GUI>
<xloc>112</xloc>
<xloc>480</xloc>
<yloc>1008</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>isLastRow</resultfieldname>
<attributes/>
<cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>112</xloc>
<yloc>912</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Neo4J Output - merge passenger-hit</name>
<type>Neo4JOutput</type>
Expand Down Expand Up @@ -941,7 +972,7 @@
</GUI>
</step>
<step>
<name>Select values</name>
<name>Select values - format dates-ids</name>
<type>SelectValues</type>
<description/>
<distribute>Y</distribute>
Expand Down Expand Up @@ -1345,7 +1376,7 @@
</output>
</remotesteps>
<GUI>
<xloc>432</xloc>
<xloc>688</xloc>
<yloc>1008</yloc>
<draw>Y</draw>
</GUI>
Expand Down Expand Up @@ -1795,8 +1826,8 @@ Aborting...gtas-passenger-hits....</logmessage>
</output>
</remotesteps>
<GUI>
<xloc>336</xloc>
<yloc>864</yloc>
<xloc>480</xloc>
<yloc>880</yloc>
<draw>Y</draw>
</GUI>
</step>
Expand Down
129 changes: 18 additions & 111 deletions gtas-neo4j-etl/job/gtas-to-neo-job.kjb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -466,45 +466,17 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<attributes_kjc/>
</entry>
<entry>
<name>Write To Log - Update Error</name>
<name>Write to log - delete temp files</name>
<description/>
<type>WRITE_TO_LOG</type>
<attributes/>
<logmessage>The gtas-update-neo4j-parameters Transformation stopped executing due to an error. </logmessage>
<loglevel>Error</loglevel>
<logsubject>ERROR-update neo4j parameter </logsubject>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>576</xloc>
<yloc>1168</yloc>
<attributes_kjc/>
</entry>
<entry>
<name>Abort job - gtas-to-neo 2</name>
<description/>
<type>ABORT</type>
<attributes/>
<message>Aborting the gtas-to-neo job due to errors in the Update Neo4J Parameters Transformation</message>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>352</xloc>
<yloc>1168</yloc>
<attributes_kjc/>
</entry>
<entry>
<name>Write to log - update parameters</name>
<description/>
<type>WRITE_TO_LOG</type>
<attributes/>
<logmessage>**STARTING TO UPDATE NEO4J PARAMETERS IN THE neo4j_parameters TABLE OF THE gtas DATABASE</logmessage>
<logmessage>** ....... DELETING TEMP FILES .......................</logmessage>
<loglevel>Minimal</loglevel>
<logsubject>log-update-neo4j-parameters</logsubject>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>832</xloc>
<xloc>848</xloc>
<yloc>1360</yloc>
<attributes_kjc/>
</entry>
Expand Down Expand Up @@ -538,8 +510,8 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>832</xloc>
<yloc>928</yloc>
<xloc>848</xloc>
<yloc>1008</yloc>
<attributes_kjc/>
</entry>
<entry>
Expand Down Expand Up @@ -732,44 +704,6 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<yloc>608</yloc>
<attributes_kjc/>
</entry>
<entry>
<name>Transformation-update parameters</name>
<description/>
<type>TRANS</type>
<attributes/>
<specification_method>filename</specification_method>
<trans_object_id/>
<filename>${Internal.Entry.Current.Directory}/gtas-update-neo4j-parameters.ktr</filename>
<transname/>
<arg_from_previous>N</arg_from_previous>
<params_from_previous>N</params_from_previous>
<exec_per_row>N</exec_per_row>
<clear_rows>N</clear_rows>
<clear_files>N</clear_files>
<set_logfile>N</set_logfile>
<logfile/>
<logext/>
<add_date>N</add_date>
<add_time>N</add_time>
<loglevel>Basic</loglevel>
<cluster>N</cluster>
<slave_server_name/>
<set_append_logfile>N</set_append_logfile>
<wait_until_finished>Y</wait_until_finished>
<follow_abort_remote>N</follow_abort_remote>
<create_parent_folder>N</create_parent_folder>
<logging_remote_work>N</logging_remote_work>
<run_configuration>Pentaho local</run_configuration>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>832</xloc>
<yloc>1168</yloc>
<attributes_kjc/>
</entry>
<entry>
<name>Transformation-check config file</name>
<description/>
Expand Down Expand Up @@ -963,8 +897,8 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>832</xloc>
<yloc>1056</yloc>
<xloc>848</xloc>
<yloc>1184</yloc>
<attributes_kjc/>
</entry>
<entry>
Expand Down Expand Up @@ -1215,7 +1149,7 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>832</xloc>
<xloc>848</xloc>
<yloc>800</yloc>
<attributes_kjc/>
</entry>
Expand Down Expand Up @@ -1350,24 +1284,6 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<evaluation>N</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Transformation-update parameters</from>
<to>Write To Log - Update Error</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Write To Log - Update Error</from>
<to>Abort job - gtas-to-neo 2</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Write to log - start-gtas-to-neo-trans</from>
<to>Transformation-gtas-to-neo</to>
Expand Down Expand Up @@ -1476,24 +1392,6 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Write to log - update parameters</from>
<to>Transformation-update parameters</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Transformation-update parameters</from>
<to>Delete Temp Files</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Delete Temp Files</from>
<to>Write To Log - success</to>
Expand Down Expand Up @@ -1649,7 +1547,7 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
</hop>
<hop>
<from>Transformation-gtas_message_staus_update</from>
<to>Write to log - update parameters</to>
<to>Write to log - delete temp files</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
Expand Down Expand Up @@ -1683,6 +1581,15 @@ Location of CONFIG FILE: ${EXT_ETL_CONFIG_FILE}</logmessage>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Write to log - delete temp files</from>
<to>Delete Temp Files</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
<notepad>
Expand Down
Loading

0 comments on commit 9946425

Please sign in to comment.