From 8814f03e6305a90d50e3ce17871480e0322d1d48 Mon Sep 17 00:00:00 2001 From: Hector Correa Date: Wed, 3 Jun 2020 13:46:11 -0400 Subject: [PATCH] Restructured the code to follow a more conventional project layout (see https://github.com/golang-standards/project-layout) --- .gitignore | 7 +++---- README.md | 15 +++++++++------ compile_all.sh => cmd/marcli/compile_all.sh | 0 {export => cmd/marcli}/json.go | 6 +++--- main.go => cmd/marcli/main.go | 11 +++++------ {export => cmd/marcli}/mrc.go | 6 +++--- {export => cmd/marcli}/mrk.go | 6 +++--- {export => cmd/marcli}/solr.go | 6 +++--- data/small.xml | 11 +++++++++++ go.mod | 2 +- {marc => pkg/marc}/field.go | 0 {marc => pkg/marc}/filters.go | 0 {marc => pkg/marc}/leader.go | 0 {marc => pkg/marc}/marcfile.go | 0 {marc => pkg/marc}/record.go | 0 {marc => pkg/marc}/xmlRecord.go | 0 16 files changed, 41 insertions(+), 29 deletions(-) rename compile_all.sh => cmd/marcli/compile_all.sh (100%) rename {export => cmd/marcli}/json.go (90%) rename main.go => cmd/marcli/main.go (78%) rename {export => cmd/marcli}/mrc.go (85%) rename {export => cmd/marcli}/mrk.go (87%) rename {export => cmd/marcli}/solr.go (96%) create mode 100644 data/small.xml rename {marc => pkg/marc}/field.go (100%) rename {marc => pkg/marc}/filters.go (100%) rename {marc => pkg/marc}/leader.go (100%) rename {marc => pkg/marc}/marcfile.go (100%) rename {marc => pkg/marc}/record.go (100%) rename {marc => pkg/marc}/xmlRecord.go (100%) diff --git a/.gitignore b/.gitignore index 7c03686..d21989e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /tmp/* -/brown/* .DS_Store # ignore the executables -marcli -marcli.exe -marcli_linux \ No newline at end of file +/cmd/marcli/marcli +/cmd/marcli/marcli.exe +/cmd/marcli/marcli_linux \ No newline at end of file diff --git a/README.md b/README.md index 15f99d9..e6787a3 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,6 @@ MARC command line (marcli) is a toy project that I am working on to help me deal The basic idea is to create a program that I can run on our Linux servers to parse our very large MARC files. The goal is to be able to copy a single file to our servers to parse these MARC files, find records that match certain criteria and export them for further review. -This program is heavily inspired by amazing work that Terry Reese has done on [MarcEdit](http://marcedit.reeset.net/). This program is not a replacement for MarcEdit. - The code is written in Go. I've found Go a very interesting and powerful programming language. Go's standard library provides most of the functionality that I need to parse MARC files. Go also supports creating binaries that can be deployed to Mac/Linux/Windows as single executable files which I love because I can deploy my executable to our Linux servers without having to do a complicated installation (i.e. no JVM needed, or Ruby bundle). @@ -41,9 +39,10 @@ You can also pass `start` and `count` parameters to output only a range of MARC ## Sample data Files under `./data/` are small MARC files that I use for testing. -* test_10.mrc has 10 MARC records -* test_1a.mrc is the first record of test_10.mrc -* test_1b.mrc is the second record of test_10.mrc +* test_10.mrc has 10 MARC records (MARC binary) +* test_1a.mrc is the first record of test_10.mrc (MARC binary) +* test_1b.mrc is the second record of test_10.mrc (MARC binary) +* small.xml has 10 MARC record (MARC XML) ## Getting started with the code @@ -51,11 +50,15 @@ Download the code and play with it: ``` git clone https://github.com/hectorcorrea/marcli.git -cd marcli +cd marcli/cmd/marcli go build ./marcli -file data/test_1a.mrc ``` +## Code Structure + +* `cmd/marcli` contains the code for the command line interface. +* `pkg/marc` contains the code to parse MARC files. ## Getting started (without the source code) If you don't care about the source code, you can download the binary file appropriated for your operating system from the [releases tab](https://github.com/hectorcorrea/marcli/releases) and run it. diff --git a/compile_all.sh b/cmd/marcli/compile_all.sh similarity index 100% rename from compile_all.sh rename to cmd/marcli/compile_all.sh diff --git a/export/json.go b/cmd/marcli/json.go similarity index 90% rename from export/json.go rename to cmd/marcli/json.go index 263dbed..60f7f1e 100644 --- a/export/json.go +++ b/cmd/marcli/json.go @@ -1,17 +1,17 @@ -package export +package main import ( "encoding/json" "errors" "fmt" + "hectorcorrea/marcli/pkg/marc" "io" - "marcli/marc" "os" ) // TODO: Add support for JSONL (JSON line delimited) format that makes JSON // easier to parse with Unix tools like grep, tail, and so on. -func ToJson(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { +func toJson(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { if len(filters.Fields) > 0 { return errors.New("filters not supported for this format") } diff --git a/main.go b/cmd/marcli/main.go similarity index 78% rename from main.go rename to cmd/marcli/main.go index 30b6ba5..6e72efd 100644 --- a/main.go +++ b/cmd/marcli/main.go @@ -4,8 +4,7 @@ import ( "errors" "flag" "fmt" - "marcli/export" - "marcli/marc" + "hectorcorrea/marcli/pkg/marc" "strings" ) @@ -33,13 +32,13 @@ func main() { searchValue := strings.ToLower(search) filters := marc.NewFieldFilters(fields) if format == "mrc" { - err = export.ToMrc(fileName, searchValue, filters, start, count) + err = toMrc(fileName, searchValue, filters, start, count) } else if format == "mrk" { - err = export.ToMrk(fileName, searchValue, filters, start, count) + err = toMrk(fileName, searchValue, filters, start, count) } else if format == "json" { - err = export.ToJson(fileName, searchValue, filters, start, count) + err = toJson(fileName, searchValue, filters, start, count) } else if format == "solr" { - err = export.ToSolr(fileName, searchValue, filters, start, count) + err = toSolr(fileName, searchValue, filters, start, count) } else { err = errors.New("Invalid format") } diff --git a/export/mrc.go b/cmd/marcli/mrc.go similarity index 85% rename from export/mrc.go rename to cmd/marcli/mrc.go index df55b25..79b222d 100644 --- a/export/mrc.go +++ b/cmd/marcli/mrc.go @@ -1,14 +1,14 @@ -package export +package main import ( "errors" "fmt" + "hectorcorrea/marcli/pkg/marc" "io" - "marcli/marc" "os" ) -func ToMrc(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { +func toMrc(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { if len(filters.Fields) > 0 { return errors.New("filters not supported for this format") } diff --git a/export/mrk.go b/cmd/marcli/mrk.go similarity index 87% rename from export/mrk.go rename to cmd/marcli/mrk.go index ed7c448..cfbc119 100644 --- a/export/mrk.go +++ b/cmd/marcli/mrk.go @@ -1,13 +1,13 @@ -package export +package main import ( "fmt" + "hectorcorrea/marcli/pkg/marc" "io" - "marcli/marc" "os" ) -func ToMrk(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { +func toMrk(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { if count == 0 { return nil } diff --git a/export/solr.go b/cmd/marcli/solr.go similarity index 96% rename from export/solr.go rename to cmd/marcli/solr.go index 8668899..1501747 100644 --- a/export/solr.go +++ b/cmd/marcli/solr.go @@ -1,11 +1,11 @@ -package export +package main import ( "encoding/json" "errors" "fmt" + "hectorcorrea/marcli/pkg/marc" "io" - "marcli/marc" "os" "strings" ) @@ -62,7 +62,7 @@ func NewSolrDocument(r marc.Record) SolrDocument { return doc } -func ToSolr(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { +func toSolr(filename string, searchValue string, filters marc.FieldFilters, start int, count int) error { if len(filters.Fields) > 0 { return errors.New("filters not supported for this format") } diff --git a/data/small.xml b/data/small.xml new file mode 100644 index 0000000..d82738a --- /dev/null +++ b/data/small.xml @@ -0,0 +1,11 @@ + + +01029cam a2200325 a 450020180816103119.0890626s1988 gw b 00000 ger d99100000035036813487090376(OCoLC)ocm19684303(OCoLC)19684303(CStRLIN)PAUG89-B23230(CaOTULAS)185174101AGW95111000000(PU)1000000-penndb-VoyagerNjPNjPgerengB491.E7S37 198809/07/89 CTZSchriften zur aristotelischen Ethik /herausgegeben von Christian Mueller-Goldingen.Hildesheim ;New York :Olms,1988.xvii, 482 p. ;23 cm.Olms StudienIn German and English.Includes bibliographical references.AristotleEthics.EthicsMueller-Goldingen, Christian.VPL09/07/89 CVPL189-B23230-1[01041 6738]09/07/89 C +01254cam a2200361 a 450020180816183115.0890626s1988 nyu b 00110 eng 9910000013503681 88026070 0824089928 (alk. paper)(OCoLC)ocm18411411(OCoLC)18411411(CStRLIN)PAUG89-B23239(CaOTULAS)185174102AGW95121000001(PU)1000001-penndb-VoyagerUPBZ8153.45.E37 1988B3216.C34016.19319B3216.C34E37 198809/07/89 CTZEggers, Walter.Ernst Cassirer :an annotated bibliography /Walter Eggers, Sigrid Mayer.New York :Garland,1988.xxiv, 483 p. ;23 cm.Garland reference library of the humanities.vol. 475Garland bibliographies of modern critics and critical schools ;vol. 13Includes index.Cassirer, Ernst,1874-1945Bibliography.Mayer, Sigrid.Garland bibliographies of modern critics and critical schools.v. 13.VPL09/07/89 CVPL189-B23239-1[01040 7471]09/07/89 C +01197cam a22003738a 450020180816183115.0890626s1989 ilu br 00110 eng 9910000023503681 88035094 1556520514 :$11.95(OCoLC)ocm19065705(OCoLC)19065705(CStRLIN)PAUG89-B23247(CaOTULAS)185174103AGW95131000002(PU)1000002-penndb-VoyagerPUHN49.V64M35 1989910/.2/0219HN49.V64M35 198909/07/89 CTZMcMillon, Bill,1942-Volunteer vacations :a directory of short-term adventures that will benefit you...and others /Bill McMillon.Rev. and expanded ed., 2nd ed.Chicago, Ill. :Chicago Review Press,c1989.8903xiv, 280 p. :ill. ;22 cm.Includes indexes.Includes bibliographical references.VoluntarismAssociations, institutions, etcDirectories.VacationsREF09/07/89 CREF189-B23247-109/07/89 C +01130cam a2200337 a 450020180816183115.0890626s1989 nyu b 00110 eng 9910000033503681 88032767 0824056434 (alk. paper)(OCoLC)ocm18815709(OCoLC)18815709(CStRLIN)PAUG89-B23248(CaOTULAS)185174104AGW95141000003(PU)1000003-penndb-VoyagerUPBUPBZ6374.B5Z78 1989DS115016.92/009292419DS115.Z82 198909/06/89 CTZZubatsky, David S.,1939-Jewish autobiographies and biographies :an international bibliography of books and dissertations in English /David S. Zubatsky.New York :Garland,1989.x, 370 p. ;23 cm.Garland reference library of the humanities.vol. 722Includes index.Bibliography: p. viii-x.JewsBiographyBibliography.REF09/06/89 CREF189-B23248-109/06/89 C +02983cas a2200601 a 450020180816150732.0820115d19782007enktr p 0 a0eng d991000003503681 90649343 sc 82003205 100966621DNLM0141-867X0141-876Xz(OCoLC)ocm04144241(OCoLC)4144241(PU)100000-penndb-VoyagerEEMEEMYUSMULHULDLCNSTAIPNSTAIPNSTIULAIPNSTNSDCOODLCNSTDLCNSDNSTNLMOCLCQCUVLVBUIUCUIZCUHULDLClce------PAUUCB411.B74940.2/53/0520Br. j. eight. century stud.British journal for eighteenth-century studiesThe British journal for eighteenth-century studies.British journal for 18th-century studiesBritish journal for eighteenth century studies[Southampton] :British Society for Eighteenth-Century Studies30 v. :ill. ;21 cm.Three no. a year,2005-Three no. a year,<1979>-1981Semiannual,1982-2004Began in spring 1978; ceased with v. 30, no. 3 (2007).Vols. for 1980-<1988> distributed by Voltaire Foundation.Description based on: Vol. 2, no. 1 (spring 1979); title from cover.Final issue consulted.Issued also on microfiche by Micromedia Division, Bell & Howell Ltd., and online.EuropeCivilization18th centuryPeriodicals.Dix-huitième sièclePériodiques.Civilisation18e sièclePériodiques.British Society for Eighteenth-Century Studies.MicroficheMicromedia Division, Bell & Howell Ltd.(DLC)sf 90092090(OCoLC)21991709British journal for eighteenth-century studies (Online)(OCoLC)60618091British Society for Eighteenth-Century Studies.Newsletter -British Society for Eighteenth-Century Studies(OCoLC)4131036Journal for eighteenth-century studies1754-0194(DLC) 2008235777(OCoLC)223475828CLUCU-ICaOLUCaOTUCtUCtYDLCDeUFUGEUICarbsInULUMHMiDWMiEMMoUNICNNCNRUNcDOCUOUPStPUTxLTWUWaUVPL104\\Current\rec'd\v.1(1978)-Currently received. Unbound issues in Current Periodicals06/11/86 C10/25/85 C06/28/84 C05/17/83 N ?105:rsl10/21/82 C02/03/82 N01/15/82 CVPL1:v.1 1978ser. Per.01/15/82 CVPL1:v.2 197901/15/82 CVPL1:v.3 198010/21/82 CVPL1:v.4 198105/04/83 CVPL1:(v.5-6 1982-83)06/28/84 CVPL1:v.7 198410/25/85 CVPL1:v.8 198506/11/86 CC0PAU +01452cam a2200337 a 450020180816183115.0890627s1989 miua b 10010 eng d99100000435036810876502524(OCoLC)ocm20435309(OCoLC)20435309(CStRLIN)PAUG89-B23411(CaOTULAS)185174105AGW95151000004(PU)1000004-penndb-VoyagerIaUIaUZ711.2.L7325 198709/06/89 CTZLibrary Instruction Conference.(14th :1987 :Ohio State University)Defining and applying effective teaching strategies for library instruction :papers presented at the fourteenth Library Instruction Conference held at Ohio State University, 7 & 8 May 1987 /edited by Mary Beth Bunge and Teresa B. Mensching.Ann Arbor, Mich. :Published for Learning Resources and Technologies, Eastern Michigan University by Pierian Press,1989.vii, 133 p. :ill. ;28 cm.Library orientation series ;no. 18Bibliography: p. 116-124.Library orientationCongresses.Bunge, Mary Beth.Mensching, Teresa B.Eastern Michigan University.Learning Resources and Technologies.Library orientation series.v. 18.VPL09/06/89 CVPL189-B23411-1[01040 5293]09/06/89 C +01400cam a2200325 a 450020180816110030.0890627s1988 miua b 10000 eng d99100000535036810876502508(OCoLC)ocm20894130(OCoLC)20894130(CStRLIN)PAUG89-B23412(CaOTULAS)185174106AGW95161000005(PU)1000005-penndb-VoyagerIaUIaUZ699.T34 198809/06/89 CTZTeaching the online catalog user /edited by Carolyn Kirkendall.Ann Arbor, Mich. :Published for the Center of Educational Resources, Eastern Michigan University by Pieran Press,1988.vii, 256 p. :ill. ;28 cm.Library orientation series ;no. 16"Papers and work session notes presented at the second Biennial LOEX Library Instruction Workshop held at Eastern Michigan University, 9 & 10 May 1985 and numerous examples of current instructional materials collected in late 1987."Bibliography: p. 227-248.Online library catalogsUser education.Kirkendall, Carolyn A.Eastern Michigan University.Center of Educational Resources.Library orientation series.v. 16.VPL09/06/89 CVPL189-B23412-1[01040 5699]09/06/89 C +01090cam a2200349 a 450020180816183115.0890627s1989 maua b 00110 eng 9910000063503681 88037315 08057799810805780475 (pbk.)(OCoLC)ocm18834791(OCoLC)18834791(CStRLIN)PAUG89-B23436(CaOTULAS)185174107AGW95171000006(PU)1000006-penndb-VoyagerNhDNjRPS3537.T3234G88 1989813/.5219PS3537.T3234G88 198909/07/89 CTZOwens, Louis.The grapes of wrath :trouble in the promised land /Louis Owens.Boston :Twayne,c1989.xv, 121 p. :ill. ;23 cm.Twayne's masterwork studies.no. 27Includes index.Bibliography: p. 111-117.Steinbeck, John,1902-1968.Grapes of wrath.VPL09/07/89 CVPL189-B23436-1[01010 2544]09/07/89 C + \ No newline at end of file diff --git a/go.mod b/go.mod index df65be5..28c9344 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module marcli +module hectorcorrea/marcli go 1.14 diff --git a/marc/field.go b/pkg/marc/field.go similarity index 100% rename from marc/field.go rename to pkg/marc/field.go diff --git a/marc/filters.go b/pkg/marc/filters.go similarity index 100% rename from marc/filters.go rename to pkg/marc/filters.go diff --git a/marc/leader.go b/pkg/marc/leader.go similarity index 100% rename from marc/leader.go rename to pkg/marc/leader.go diff --git a/marc/marcfile.go b/pkg/marc/marcfile.go similarity index 100% rename from marc/marcfile.go rename to pkg/marc/marcfile.go diff --git a/marc/record.go b/pkg/marc/record.go similarity index 100% rename from marc/record.go rename to pkg/marc/record.go diff --git a/marc/xmlRecord.go b/pkg/marc/xmlRecord.go similarity index 100% rename from marc/xmlRecord.go rename to pkg/marc/xmlRecord.go