5.1.0.2

legumeinfo · Jan 11, 2023 · 3bc47d2 · 3bc47d2
1 parent e403f12
commit 3bc47d2
Show file tree

Hide file tree

Showing 27 changed files with 168 additions and 101 deletions.
diff --git a/build.gradle b/build.gradle
@@ -21,7 +21,7 @@ ext {
 
 subprojects {
     group = 'org.intermine'
-    version = '5.1.0.1'
+    version = '5.1.0.2'
 
     apply plugin: 'java'
     apply plugin: 'maven'

diff --git a/gradle.properties b/gradle.properties
@@ -1,2 +1,2 @@
-systemProp.imVersion=5.1.0.1
-systemProp.bioVersion=5.1.0.1
+systemProp.imVersion=5.1.0.2
+systemProp.bioVersion=5.1.0.2
diff --git a/lis-annotation/libs/ncgr-datastore.jar b/lis-annotation/libs/ncgr-datastore.jar
diff --git a/lis-annotation/src/main/java/org/intermine/bio/dataconversion/AnnotationFileConverter.java b/lis-annotation/src/main/java/org/intermine/bio/dataconversion/AnnotationFileConverter.java
@@ -228,6 +228,33 @@ public void close() throws ObjectStoreException, RuntimeException {
             if (cdses.containsKey(primaryIdentifier)) protein.setReference("CDS", cdses.get(primaryIdentifier));
             if (mRNAs.containsKey(primaryIdentifier)) protein.setReference("transcript", mRNAs.get(primaryIdentifier));
         }
+        // add publication to all Annotatables
+        if (publication!=null) {
+            for (Item chromosome : chromosomes.values()) {
+                chromosome.addToCollection("publications", publication);
+            }
+            for (Item supercontig : supercontigs.values()) {
+                supercontig.addToCollection("publications", publication);
+            }
+            for (Item feature : features.values()) {
+                feature.addToCollection("publications", publication);
+            }
+            for (Item cds : cdses.values()) {
+                cds.addToCollection("publications", publication);
+            }
+            for (Item gene : genes.values()) {
+                gene.addToCollection("publications", publication);
+            }
+            for (Item mRNA : mRNAs.values()) {
+                mRNA.addToCollection("publications", publication);
+            }
+            for (Item protein : proteins.values()) {
+                protein.addToCollection("publications", publication);
+            }
+            for (Item proteinDomain : proteinDomains.values()) {
+                proteinDomain.addToCollection("publications", publication);
+            }
+        }
         // store
         storeCollectionItems();
         store(chromosomes.values());

diff --git a/lis-expression/libs/ncgr-datastore.jar b/lis-expression/libs/ncgr-datastore.jar
diff --git a/lis-expression/src/main/java/org/intermine/bio/dataconversion/ExpressionFileConverter.java b/lis-expression/src/main/java/org/intermine/bio/dataconversion/ExpressionFileConverter.java
@@ -7,7 +7,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
-import java.util.LinkedList;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.HashMap;
 
@@ -38,22 +38,16 @@ public class ExpressionFileConverter extends DatastoreFileConverter {
 
     // README items
     Item expressionSource;
-    String expressionUnit;
 
     // Lists
-    List<Item> expressionValues = new LinkedList<>();
-    List<Item> annotations = new LinkedList<>();
+    List<Item> expressionValues = new ArrayList<>();
+    List<Item> ontologyAnnotations = new ArrayList<>();
 
     // Maps
-    Map<String,Item> terms = new HashMap<>();
+    Map<String,Item> ontologyTerms = new HashMap<>();
     Map<String,Item> samples = new HashMap<>();
     Map<String,Item> genes = new HashMap<>();
 
-    // keep track of files read in case they're gzipped!
-    boolean samplesRead = false;
-    boolean valuesRead = false;
-    boolean oboRead = false;
-
     // validate the collection first by storing a flag
     boolean collectionValidated = false;
 
@@ -95,25 +89,22 @@ public void process(Reader reader) throws IOException {
             expressionSource.setAttribute("primaryIdentifier", readme.identifier);
             expressionSource.setAttribute("synopsis", readme.synopsis);
             expressionSource.setAttribute("description", readme.description);
+            expressionSource.setAttribute("unit", readme.expression_unit);
             expressionSource.setReference("organism", organism);
             expressionSource.setReference("strain", strain);
-            expressionUnit = readme.expression_unit; // applied to ExpressionValue.unit in close()
             if (readme.geoseries!=null) expressionSource.setAttribute("geoSeries", readme.geoseries);
             if (readme.sraproject!=null) expressionSource.setAttribute("sra", readme.sraproject);
             if (readme.bioproject!=null) expressionSource.setAttribute("bioProject", readme.bioproject);
             if (publication!=null) expressionSource.addToCollection("publications", publication);
         } else if (getCurrentFile().getName().endsWith("samples.tsv.gz")) {
             System.out.println("## Processing "+getCurrentFile().getName());
 	    processSamplesFile();
-            samplesRead = true;
         } else if (getCurrentFile().getName().endsWith("values.tsv.gz")) {
             System.out.println("## Processing "+getCurrentFile().getName());
             processValuesFile();
-            valuesRead = true;
         } else if (getCurrentFile().getName().endsWith("obo.tsv.gz")) {
             System.out.println("## Processing "+getCurrentFile().getName());
             processOboFile();
-            oboRead = true;
         } else {
             System.out.println("## - Skipping "+getCurrentFile().getName());
         }
@@ -124,9 +115,6 @@ public void process(Reader reader) throws IOException {
      */
     @Override
     public void close() throws ObjectStoreException {
-        if (!samplesRead || !valuesRead || !oboRead) {
-            throw new RuntimeException("One of samples.tsv.gz, values.tsv.gz, and/or obo.tsv.gz files not read. Aborting.");
-        }
         // DatastoreFileConverter items
         storeCollectionItems();
         // add references to samples
@@ -142,8 +130,8 @@ public void close() throws ObjectStoreException {
         store(expressionSource);
 	store(genes.values());
         store(samples.values());
-        store(terms.values());
-        store(annotations);
+        store(ontologyTerms.values());
+        store(ontologyAnnotations);
 	store(expressionValues);
     }
 
@@ -212,7 +200,7 @@ void processSamplesFile() throws IOException {
      * cajca.ICPL87119.gnm1.ann1.C.cajan_00002	0	0	0	0	...
      */
     void processValuesFile() throws IOException {
-        List<Item> sampleList = new LinkedList<>();
+        List<Item> sampleList = new ArrayList<>();
 	BufferedReader br = GZIPBufferedReader.getReader(getCurrentFile());
         String line = null;
         while ((line=br.readLine())!=null) {
@@ -230,7 +218,6 @@ void processValuesFile() throws IOException {
                     double value = Double.parseDouble(parts[i]);
                     Item sample = sampleList.get(i-1);
                     Item expressionValue = createItem("ExpressionValue");
-                    expressionValue.setAttribute("unit", expressionUnit);
                     expressionValue.setAttribute("value", String.valueOf(value));
                     expressionValue.setReference("feature", gene);
                     expressionValue.setReference("sample", sample);
@@ -253,20 +240,14 @@ void processOboFile() throws IOException {
             String sampleId = parts[0];
             String termId = parts[1];
             Item sample = getSample(sampleId);
-            Item term = terms.get(termId);
-            if (term==null) {
-                term = createItem("OntologyTerm");
-                term.setAttribute("identifier", termId);
-                terms.put(termId, term);
-            }
-            Item annotation = createItem("OntologyAnnotation");
-            annotation.setReference("subject", sample);
-            annotation.setReference("ontologyTerm", term);
-            annotations.add(annotation);
+            Item ontologyTerm = getOntologyTerm(termId);
+            Item ontologyAnnotation = createItem("OntologyAnnotation");
+            ontologyAnnotation.setReference("subject", sample);
+            ontologyAnnotation.setReference("ontologyTerm", ontologyTerm);
+            ontologyAnnotations.add(ontologyAnnotation);
         }
     }
 
-
     /**
      * Get or create a Sample.
      */
@@ -275,7 +256,7 @@ Item getSample(String id) {
             return samples.get(id);
         } else {
             Item sample = createItem("ExpressionSample");
-            sample.setAttribute("identifier", id);
+            sample.setAttribute("primaryIdentifier", id);
             samples.put(id, sample);
             return sample;
         }
@@ -284,14 +265,14 @@ Item getSample(String id) {
     /**
      * Get or create an OntologyTerm.
      */
-    Item getOboTerm(String id) {
-        if (terms.containsKey(id)) {
-            return terms.get(id);
+    Item getOntologyTerm(String id) {
+        if (ontologyTerms.containsKey(id)) {
+            return ontologyTerms.get(id);
         } else {
-            Item term = createItem("OntologyTerm");
-            term.setAttribute("identifier", id);
-            terms.put(id, term);
-            return term;
+            Item ontologyTerm = createItem("OntologyTerm");
+            ontologyTerm.setAttribute("identifier", id);
+            ontologyTerms.put(id, ontologyTerm);
+            return ontologyTerm;
         }
     }
 
@@ -309,4 +290,6 @@ Item getGene(String id) {
         }
     }
 
+
+
 }
diff --git a/lis-expression/src/main/resources/lis-expression_keys.properties b/lis-expression/src/main/resources/lis-expression_keys.properties
@@ -15,12 +15,8 @@ Strain.key_identifier=identifier
 Publication.key_doi=doi
 
 ## Annotatable
+ExpressionSample.key_primaryidentifier=primaryIdentifier
 ExpressionSource.key_primaryidentifier=primaryIdentifier
 
 ## BioEntity
 SequenceFeature.key_primaryidentifier=primaryIdentifier
-
-
-## Other
-ExpressionSample.key_identifier=identifier
-
diff --git a/lis-genefamily/src/main/java/org/intermine/bio/dataconversion/GeneFamilyFileConverter.java b/lis-genefamily/src/main/java/org/intermine/bio/dataconversion/GeneFamilyFileConverter.java
@@ -75,6 +75,18 @@ public void close() throws Exception {
         if (!ahrdFileFound) {
             throw new RuntimeException("File ending in .info_annot_ahrd.tsv.gz not found. Aborting");
         }
+        // add publication to Annotatables
+        if (publication!=null) {
+            for (Item protein : proteins.values()) {
+                protein.addToCollection("publications", publication);
+            }
+            for (Item gene : genes.values()) {
+                gene.addToCollection("publications", publication);
+            }
+            for (Item proteinDomain : proteinDomains.values()) {
+                proteinDomain.addToCollection("publications", publication);
+            }
+        }
         storeCollectionItems();
 	store(geneFamilies.values());
 	store(ontologyTerms.values());

diff --git a/lis-genome/libs/ncgr-datastore.jar b/lis-genome/libs/ncgr-datastore.jar
diff --git a/lis-genome/src/main/java/org/intermine/bio/dataconversion/GenomeFastaLoaderTask.java b/lis-genome/src/main/java/org/intermine/bio/dataconversion/GenomeFastaLoaderTask.java
@@ -375,6 +375,7 @@ void storeSequenceFeature(SequenceFeature feature, Sequence bioSequence) throws
         feature.setStrain(strain);
         feature.setSequence(bioSequence);
         feature.setLength(bioSequence.getLength());
+        if (publication!=null) feature.addPublications(publication);
         getDirectDataLoader().store(bioSequence);
         getDirectDataLoader().store(feature);
     }

diff --git a/lis-gwas/libs/ncgr-datastore.jar b/lis-gwas/libs/ncgr-datastore.jar
diff --git a/lis-gwas/src/main/java/org/intermine/bio/dataconversion/GWASFileConverter.java b/lis-gwas/src/main/java/org/intermine/bio/dataconversion/GWASFileConverter.java
@@ -5,7 +5,7 @@
 import java.io.Reader;
 
 import java.util.List;
-import java.util.LinkedList;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.HashMap;
 
@@ -34,10 +34,14 @@ public class GWASFileConverter extends DatastoreFileConverter {
 
     // local Items to store
     Item gwas;
-    List<Item> gwasResults = new LinkedList<>();
+    List<Item> gwasResults = new ArrayList<>();
+    List<Item> ontologyAnnotations = new ArrayList<>();
     Map<String,Item> ontologyTerms = new HashMap<>();
     Map<String,Item> traits = new HashMap<>();
 
+    // utility
+    String collectionIdentifier;
+
     // validate the collection first by storing a flag
     boolean collectionValidated = false;
 
@@ -71,9 +75,10 @@ public void process(Reader reader) throws IOException {
         }
         if (getCurrentFile().getName().startsWith("README")) {
             processReadme(reader);
+            collectionIdentifier = readme.identifier;
             gwas = createItem("GWAS");
             gwas.setReference("organism", organism);
-            gwas.setAttribute("primaryIdentifier", readme.identifier);
+            gwas.setAttribute("primaryIdentifier", collectionIdentifier);
             gwas.setAttribute("synopsis", readme.synopsis);
             gwas.setAttribute("description", readme.description);
             gwas.setAttribute("genotypingPlatform", readme.genotyping_platform);
@@ -125,6 +130,7 @@ public void close() throws ObjectStoreException {
         store(gwasResults);
         store(traits.values());
         store(ontologyTerms.values());
+        store(ontologyAnnotations);
     }
 
     /**
@@ -178,8 +184,11 @@ void processOboFile() throws IOException {
             Item trait = getTrait(traitName);
             // 1:OntologyTerm
             Item ontologyTerm = getOntologyTerm(oboTerm);
-            // add ontology term
-            trait.addToCollection("ontologyTerms", ontologyTerm);
+            // add ontology annotation
+            Item ontologyAnnotation = createItem("OntologyAnnotation");
+            ontologyAnnotation.setReference("subject", trait);
+            ontologyAnnotation.setReference("ontologyTerm", ontologyTerm);
+            ontologyAnnotations.add(ontologyAnnotation);
         }
         br.close();
     }
@@ -222,14 +231,15 @@ void processResultFile() throws IOException {
     }
 
     /**
-     * Return a new or existing Trait Item keyed by name.
+     * Return a new or existing Trait Item keyed by name; primaryIdentifier is concocted from collection identifier and name.
      */
     Item getTrait(String name) {
         if (traits.containsKey(name)) {
             return traits.get(name);
         } else {
             Item trait = createItem("Trait");
             trait.setAttribute("name", name);
+            trait.setAttribute("primaryIdentifier", collectionIdentifier+":"+name);
             traits.put(name, trait);
             return trait;
         }

diff --git a/lis-gwas/src/main/resources/lis-gwas_keys.properties b/lis-gwas/src/main/resources/lis-gwas_keys.properties
@@ -15,6 +15,4 @@ Publication.key_doi=doi
 
 ## Annotatable
 GWAS.key_primaryidentifier=primaryIdentifier
-
-## non-Annottable
-Trait.key_name_gwas=name,gwas
+Trait.key_primaryidentifier=primaryIdentifier
diff --git a/lis-hsh/src/main/java/org/intermine/bio/dataconversion/HSHFileConverter.java b/lis-hsh/src/main/java/org/intermine/bio/dataconversion/HSHFileConverter.java
@@ -72,6 +72,15 @@ public void process(Reader reader) throws IOException {
     public void close() throws ObjectStoreException {
         // standard collection items
         storeCollectionItems();
+        // add publication to Annotatables
+        if (publication!=null) {
+            for (Item gene : genes.values()) {
+                gene.addToCollection("publications", publication);
+            }
+            for (Item protein : proteins.values()) {
+                protein.addToCollection("publications", publication);
+            }
+        }
         // local items
         store(panGeneSets.values());
         store(genes.values());

diff --git a/lis-info-annot/src/main/java/org/intermine/bio/dataconversion/InfoAnnotFileConverter.java b/lis-info-annot/src/main/java/org/intermine/bio/dataconversion/InfoAnnotFileConverter.java
@@ -130,6 +130,18 @@ public void close() throws ObjectStoreException {
             mRNA.setReference("organism", organism);
             mRNA.setReference("strain", strain);
         }
+        // add publication to Annotatables
+        if (publication!=null) {
+            for (Item gene : genes.values()) {
+                gene.addToCollection("publications", publication);
+            }
+            for (Item protein : proteins.values()) {
+                protein.addToCollection("publications", publication);
+            }
+            for (Item mRNA : mRNAs.values()) {
+                mRNA.addToCollection("publications", publication);
+            }
+        }
         // store standard collection Items
         storeCollectionItems();
         // store local items

diff --git a/lis-map/libs/ncgr-datastore.jar b/lis-map/libs/ncgr-datastore.jar
diff --git a/lis-map/src/main/resources/lis-map_keys.properties b/lis-map/src/main/resources/lis-map_keys.properties
@@ -1,5 +1,5 @@
 ##
-## legfed-map keys
+## lis-map keys
 ##
 
 ## DatastoreFileConverter

diff --git a/lis-markers/libs/ncgr-datastore.jar b/lis-markers/libs/ncgr-datastore.jar
diff --git a/lis-markers/src/main/java/org/intermine/bio/dataconversion/MarkerGFF3FileConverter.java b/lis-markers/src/main/java/org/intermine/bio/dataconversion/MarkerGFF3FileConverter.java
@@ -125,6 +125,12 @@ public void close() throws ObjectStoreException {
             supercontig.setReference("organism", organism);
             supercontig.setReference("strain", strain);
         }
+        // add publication to Annotatables (but not chromosome/supercontig)
+        if (publication!=null) {
+            for (Item geneticMarker : geneticMarkers.values()) {
+                geneticMarker.addToCollection("publications", publication);
+            }
+        }
         storeCollectionItems();
         // local items
         store(chromosomes.values());

diff --git a/lis-mstmap/libs/ncgr-datastore.jar b/lis-mstmap/libs/ncgr-datastore.jar