Skip to content

Commit

Permalink
added loading ModuleDependency files
Browse files Browse the repository at this point in the history
  • Loading branch information
Timur committed Nov 5, 2020
1 parent 83242dc commit e1282df
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 19 deletions.
19 changes: 16 additions & 3 deletions SNOMED/create_source_tables.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ CREATE TABLE SOURCES.DER2_CREFSET_ASSREFFULL_MERGED
(
ID VARCHAR(256),
EFFECTIVETIME VARCHAR(8),
ACTIVE INTEGER,
ACTIVE INTEGER,
MODULEID BIGINT,
REFSETID BIGINT,
REFERENCEDCOMPONENTID BIGINT,
Expand All @@ -75,7 +75,7 @@ CREATE TABLE SOURCES.DER2_SREFSET_SIMPLEMAPFULL_INT
(
ID VARCHAR(256),
EFFECTIVETIME VARCHAR(8),
ACTIVE INTEGER,
ACTIVE INTEGER,
MODULEID BIGINT,
REFSETID BIGINT,
REFERENCEDCOMPONENTID BIGINT,
Expand All @@ -87,14 +87,27 @@ CREATE TABLE SOURCES.DER2_CREFSET_LANGUAGE_MERGED
(
ID VARCHAR(256),
EFFECTIVETIME VARCHAR(8),
ACTIVE INTEGER,
ACTIVE INTEGER,
MODULEID BIGINT,
REFSETID BIGINT,
REFERENCEDCOMPONENTID BIGINT,
ACCEPTABILITYID BIGINT,
SOURCE_FILE_ID VARCHAR(10)
);

DROP TABLE IF EXISTS SOURCES.DER2_SSREFSET_MODULEDEPENDENCY_MERGED;
CREATE TABLE SOURCES.DER2_SSREFSET_MODULEDEPENDENCY_MERGED
(
ID VARCHAR(256),
EFFECTIVETIME VARCHAR(8),
ACTIVE INTEGER,
MODULEID BIGINT,
REFSETID BIGINT,
REFERENCEDCOMPONENTID BIGINT,
SOURCEEFFECTIVETIME DATE,
TARGETEFFECTIVETIME DATE
);

CREATE INDEX idx_concept_merged_id ON SOURCES.SCT2_CONCEPT_FULL_MERGED (ID);
CREATE INDEX idx_desc_merged_id ON SOURCES.SCT2_DESC_FULL_MERGED (CONCEPTID);
CREATE INDEX idx_rela_merged_id ON SOURCES.SCT2_RELA_FULL_MERGED (ID);
Expand Down
16 changes: 12 additions & 4 deletions SNOMED/readme.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ from the folder \Full\Refset\Map:
- der2_sRefset_SimpleMapFull_INT_YYYYMMDD.txt
from the folder \Full\Refset\Language
- der2_cRefset_LanguageFull-en_INT_YYYYMMDD.txt
Rename files to sct2_Concept_Full_INT.txt, sct2_Description_Full-en_INT.txt, sct2_Relationship_Full_INT.txt, der2_sRefset_SimpleMapFull_INT.txt, der2_cRefset_LanguageFull_INT.txt
from the folder \Full\Refset\Metadata
- der2_ssRefset_ModuleDependencyFull_INT_YYYYMMDD.txt
Rename files to sct2_Concept_Full_INT.txt, sct2_Description_Full-en_INT.txt, sct2_Relationship_Full_INT.txt, der2_sRefset_SimpleMapFull_INT.txt, der2_cRefset_LanguageFull_INT.txt, der2_ssRefset_ModuleDependencyFull_INT.txt

4. Download the British SNOMED file uk_sct2cl_xx.x.x__YYYYMMDD000001.zip from https://isd.digital.nhs.uk/trud3/user/authenticated/group/0/pack/26/subpack/101/releases.
5. Extract the following files from the folder SnomedCT_UKClinicalRF2_Production_YYYYMMDDTzzzzzz\Full\Terminology into a working folder:
Expand All @@ -24,7 +26,9 @@ Rename files to sct2_Concept_Full_INT.txt, sct2_Description_Full-en_INT.txt, sct
- sct2_Relationship_Full-GB_GB1000000_YYYYMMDD.txt
from the folder \Full\Refset\Language
- der2_cRefset_LanguageFull-en-GB_GB1000000_YYYYMMDD.txt
Rename files to sct2_Concept_Full-UK.txt, sct2_Description_Full-UK.txt, sct2_Relationship_Full-UK.txt, der2_cRefset_LanguageFull_UK.txt
from the folder \Full\Refset\Metadata
- der2_ssRefset_ModuleDependencyFull_GB1000000_YYYYMMDD.txt
Rename files to sct2_Concept_Full-UK.txt, sct2_Description_Full-UK.txt, sct2_Relationship_Full-UK.txt, der2_cRefset_LanguageFull_UK.txt, der2_ssRefset_ModuleDependencyFull_UK.txt

6. Download the US SNOMED file SnomedCT_USEditionRF2_PRODUCTION_YYYYMMDDTzzzzzzZ.zip from https://www.nlm.nih.gov/healthit/snomedct/us_edition.html
7. Extract the following files from the folder \Full\Terminology\ into a working folder:
Expand All @@ -33,7 +37,9 @@ Rename files to sct2_Concept_Full-UK.txt, sct2_Description_Full-UK.txt, sct2_Rel
- sct2_Relationship_Full_US1000124_YYYYMMDD.txt
from the folder \Full\Refset\Language
- der2_cRefset_LanguageFull-en_US1000124_YYYYMMDD.txt
Remove date from file name and rename to sct2_Concept_Full_US.txt, sct2_Description_Full-en_US.txt, sct2_Relationship_Full_US.txt, der2_cRefset_LanguageFull_US.txt
from the folder \Full\Refset\Metadata
- der2_ssRefset_ModuleDependencyFull_US1000124_YYYYMMDD.txt
Remove date from file name and rename to sct2_Concept_Full_US.txt, sct2_Description_Full-en_US.txt, sct2_Relationship_Full_US.txt, der2_cRefset_LanguageFull_US.txt, der2_ssRefset_ModuleDependencyFull_US.txt

8. Download the UK SNOMED CT Drug Extension, RF2 file uk_sct2dr_xx.x.x__YYYYMMDD000001.zip from https://isd.digital.nhs.uk/trud3/user/authenticated/group/0/pack/26/subpack/105/releases
9. Extract the following files from the folder SnomedCT_UKDrugRF2_Production_20180516T000001Z\Full\Terminology\ into a working folder:
Expand All @@ -42,7 +48,9 @@ Remove date from file name and rename to sct2_Concept_Full_US.txt, sct2_Descript
- sct2_Relationship_Full_GB1000000_YYYYMMDD.txt
from the folder \Full\Refset\Language
- der2_cRefset_LanguageFull-en-GB_GB1000001_YYYYMMDD.txt
Rename files to sct2_Concept_Full_GB_DE.txt, sct2_Description_Full-en-GB_DE.txt, sct2_Relationship_Full_GB_DE.txt, der2_cRefset_LanguageFull_GB_DE.txt
from the folder \Full\Refset\Metadata
- der2_ssRefset_ModuleDependencyFull_GB1000001_YYYYMMDD.txt
Rename files to sct2_Concept_Full_GB_DE.txt, sct2_Description_Full-en-GB_DE.txt, sct2_Relationship_Full_GB_DE.txt, der2_cRefset_LanguageFull_GB_DE.txt, der2_ssRefset_ModuleDependencyFull_GB_DE.txt

10. Extract
- der2_cRefset_AssociationFull_INT_YYYYMMDD.txt from SnomedCT_InternationalRF2_Production_YYYYMMDDTzzzzzz\Full\Refset\Content
Expand Down
6 changes: 6 additions & 0 deletions working/packages/load_input_tables/load_input_tables.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,12 @@ begin
update sources.der2_crefset_language_merged set source_file_id='GB_DE' where source_file_id is null;
CREATE INDEX idx_lang_merged_refid ON sources.der2_crefset_language_merged (referencedcomponentid);
analyze sources.der2_crefset_language_merged;
--loading der2_ssrefset_moduledependency_merged
truncate table sources.der2_ssrefset_moduledependency_merged;
execute 'COPY sources.der2_ssrefset_moduledependency_merged FROM '''||pVocabularyPath||'der2_ssRefset_ModuleDependencyFull_INT.txt'' delimiter E''\t'' csv quote E''\b'' HEADER';
execute 'COPY sources.der2_ssrefset_moduledependency_merged FROM '''||pVocabularyPath||'der2_ssRefset_ModuleDependencyFull_UK.txt'' delimiter E''\t'' csv quote E''\b'' HEADER';
execute 'COPY sources.der2_ssrefset_moduledependency_merged FROM '''||pVocabularyPath||'der2_ssRefset_ModuleDependencyFull_US.txt'' delimiter E''\t'' csv quote E''\b'' HEADER';
execute 'COPY sources.der2_ssrefset_moduledependency_merged FROM '''||pVocabularyPath||'der2_ssRefset_ModuleDependencyFull_GB_DE.txt'' delimiter E''\t'' csv quote E''\b'' HEADER';
when 'ICD10CM' then
truncate table sources.icd10cm_temp, sources.icd10cm;
execute 'COPY sources.icd10cm_temp FROM '''||pVocabularyPath||'icd10cm.txt'' delimiter E''\b''';
Expand Down
33 changes: 21 additions & 12 deletions working/packages/vocabulary_download/bash_functions_snomed.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
20190204 added flag -C to unzip (only SNOMED part) = case insensitive
20200225 added extraction of der2_Refset_SimpleFull_INT.txt
20200519 added extraction of der2_cRefset_LanguageFull.txt
20200511 added extraction of der2_ssRefset_ModuleDependencyFull*.txt
*/

--INT part
Expand All @@ -21,17 +22,19 @@ BEGIN
unzip -oqjC "$2" "SnomedCT_InternationalRF2_PRODUCTION_*/Full/Terminology/sct2_Relationship_Full_INT_*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_InternationalRF2_PRODUCTION_*/Full/Refset/Content/der2_cRefset_AssociationFull_INT_*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_InternationalRF2_PRODUCTION_*/Full/Refset/Map/der2_sRefset_SimpleMapFull_INT_*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_InternationalRF2_PRODUCTION_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d .
unzip -oqjC "$2" "SnomedCT_InternationalRF2_PRODUCTION_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_InternationalRF2_PRODUCTION_*/Full/Refset/Metadata/der2_ssRefset_ModuleDependencyFull*.txt" -d .

#move result to original folder
cd "$1"
rm -f "sct2_Concept_Full_INT.txt" "sct2_Description_Full-en_INT.txt" "sct2_Relationship_Full_INT.txt" "der2_cRefset_AssociationFull_INT.txt" "der2_sRefset_SimpleMapFull_INT.txt" "der2_sRefset_LanguageFull_INT.txt"
rm -f "sct2_Concept_Full_INT.txt" "sct2_Description_Full-en_INT.txt" "sct2_Relationship_Full_INT.txt" "der2_cRefset_AssociationFull_INT.txt" "der2_sRefset_SimpleMapFull_INT.txt" "der2_sRefset_LanguageFull_INT.txt" "der2_ssRefset_ModuleDependencyFull_INT.txt"
mv work/sct2_Concept_Full_INT_*.txt "sct2_Concept_Full_INT.txt" && \
mv work/sct2_Description_Full-en_INT_*.txt "sct2_Description_Full-en_INT.txt" && \
mv work/sct2_Relationship_Full_INT_*.txt "sct2_Relationship_Full_INT.txt" && \
mv work/der2_cRefset_AssociationFull_INT_*.txt "der2_cRefset_AssociationFull_INT.txt" && \
mv work/der2_sRefset_SimpleMapFull_INT_*.txt "der2_sRefset_SimpleMapFull_INT.txt" && \
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_INT.txt"
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_INT.txt" && \
mv work/der2_ssRefset_ModuleDependencyFull*.txt "der2_ssRefset_ModuleDependencyFull_INT.txt"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
Expand All @@ -56,16 +59,18 @@ BEGIN
unzip -oqjC "$2" "SnomedCT_UKClinicalRF2_PRODUCTION_*/Full/Terminology/sct2_Description_Full*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKClinicalRF2_PRODUCTION_*/Full/Terminology/sct2_Relationship_Full*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKClinicalRF2_PRODUCTION_*/Full/Refset/Content/der2_cRefset_AssociationFull_GB*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKClinicalRF2_PRODUCTION_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d .
unzip -oqjC "$2" "SnomedCT_UKClinicalRF2_PRODUCTION_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKClinicalRF2_PRODUCTION_*/Full/Refset/Metadata/der2_ssRefset_ModuleDependencyFull*.txt" -d .

#move result to original folder
cd "$1"
rm -f "sct2_Concept_Full-UK.txt" "sct2_Description_Full-UK.txt" "sct2_Relationship_Full-UK.txt" "der2_cRefset_AssociationFull_UK.txt" "der2_sRefset_LanguageFull_UK.txt"
rm -f "sct2_Concept_Full-UK.txt" "sct2_Description_Full-UK.txt" "sct2_Relationship_Full-UK.txt" "der2_cRefset_AssociationFull_UK.txt" "der2_sRefset_LanguageFull_UK.txt" "der2_ssRefset_ModuleDependencyFull_UK.txt"
mv work/sct2_Concept_Full*.txt "sct2_Concept_Full-UK.txt" && \
mv work/sct2_Description_Full*.txt "sct2_Description_Full-UK.txt" && \
mv work/sct2_Relationship_Full*.txt "sct2_Relationship_Full-UK.txt" && \
mv work/der2_cRefset_AssociationFull*.txt "der2_cRefset_AssociationFull_UK.txt" && \
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_UK.txt"
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_UK.txt" && \
mv work/der2_ssRefset_ModuleDependencyFull*.txt "der2_ssRefset_ModuleDependencyFull_UK.txt"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
Expand All @@ -90,16 +95,18 @@ BEGIN
unzip -oqjC "$2" "SnomedCT_USEditionRF2_PRODUCTION_*/Full/Terminology/sct2_Description_Full-en_US*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_USEditionRF2_PRODUCTION_*/Full/Terminology/sct2_Relationship_Full_*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_USEditionRF2_PRODUCTION_*/Full/Refset/Content/der2_cRefset_AssociationFull_US*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_USEditionRF2_PRODUCTION_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d .
unzip -oqjC "$2" "SnomedCT_USEditionRF2_PRODUCTION_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_USEditionRF2_PRODUCTION_*/Full/Refset/Metadata/der2_ssRefset_ModuleDependencyFull*.txt" -d .

#move result to original folder
cd "$1"
rm -f "sct2_Concept_Full_US.txt" "sct2_Description_Full-en_US.txt" "sct2_Relationship_Full_US.txt" "der2_cRefset_AssociationFull_US.txt" "der2_sRefset_LanguageFull_US.txt"
rm -f "sct2_Concept_Full_US.txt" "sct2_Description_Full-en_US.txt" "sct2_Relationship_Full_US.txt" "der2_cRefset_AssociationFull_US.txt" "der2_sRefset_LanguageFull_US.txt" "der2_ssRefset_ModuleDependencyFull_US.txt"
mv work/sct2_Concept_Full_US*.txt "sct2_Concept_Full_US.txt" && \
mv work/sct2_Description_Full-en_US*.txt "sct2_Description_Full-en_US.txt" && \
mv work/sct2_Relationship_Full_US*.txt "sct2_Relationship_Full_US.txt" && \
mv work/der2_cRefset_AssociationFull_US*.txt "der2_cRefset_AssociationFull_US.txt" && \
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_US.txt"
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_US.txt" && \
mv work/der2_ssRefset_ModuleDependencyFull*.txt "der2_ssRefset_ModuleDependencyFull_US.txt"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
Expand All @@ -124,16 +131,18 @@ BEGIN
unzip -oqjC "$2" "SnomedCT_UKDrugRF2_Production_*/Full/Terminology/sct2_Description_Full-en_GB*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKDrugRF2_Production_*/Full/Terminology/sct2_Relationship_Full_GB*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKDrugRF2_Production_*/Full/Refset/Content/der2_cRefset_AssociationFull_GB*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKDrugRF2_Production_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d .
unzip -oqjC "$2" "SnomedCT_UKDrugRF2_Production_*/Full/Refset/Language/der2_cRefset_LanguageFull*.txt" -d . && \
unzip -oqjC "$2" "SnomedCT_UKDrugRF2_Production_*/Full/Refset/Metadata/der2_ssRefset_ModuleDependencyFull*.txt" -d .

#move result to original folder
cd "$1"
rm -f "sct2_Concept_Full_GB_DE.txt" "sct2_Description_Full-en-GB_DE.txt" "sct2_Relationship_Full_GB_DE.txt" "der2_cRefset_AssociationFull_GB_DE.txt" "der2_sRefset_LanguageFull_GB_DE.txt"
rm -f "sct2_Concept_Full_GB_DE.txt" "sct2_Description_Full-en-GB_DE.txt" "sct2_Relationship_Full_GB_DE.txt" "der2_cRefset_AssociationFull_GB_DE.txt" "der2_sRefset_LanguageFull_GB_DE.txt" "der2_ssRefset_ModuleDependencyFull_GB_DE.txt"
mv work/sct2_Concept_Full_GB*.txt "sct2_Concept_Full_GB_DE.txt" && \
mv work/sct2_Description_Full-en_GB*.txt "sct2_Description_Full-en-GB_DE.txt" && \
mv work/sct2_Relationship_Full_GB*.txt "sct2_Relationship_Full_GB_DE.txt" && \
mv work/der2_cRefset_AssociationFull_GB*.txt "der2_cRefset_AssociationFull_GB_DE.txt" && \
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_GB_DE.txt"
mv work/der2_cRefset_LanguageFull*.txt "der2_sRefset_LanguageFull_GB_DE.txt" && \
mv work/der2_ssRefset_ModuleDependencyFull*.txt "der2_ssRefset_ModuleDependencyFull_GB_DE.txt"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
Expand Down

0 comments on commit e1282df

Please sign in to comment.