Skip to content

Commit

Permalink
added bidirectional grad score - it will not be set as not computed s…
Browse files Browse the repository at this point in the history 
…ince this branch still computes only 2D grad scores
  • Loading branch information
Cristian Goina committed Sep 13, 2024
1 parent 02f3c6d commit 06fcabf
Show file tree
Hide file tree
Showing 6 changed files with 97 additions and 107 deletions.
42 changes: 20 additions & 22 deletions colormipsearch-api/src/main/java/org/janelia/colormipsearch/cds/GradientAreaGapUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ else if (colorB > colorG)
return sliceNumber;
}

public static long calculateNegativeScore(Long gradientAreaGap, Long highExpressionArea) {
public static long calculate2DShapeScore(Long gradientAreaGap, Long highExpressionArea) {
long negativeScore;
if (gradientAreaGap != null && gradientAreaGap >= 0 && highExpressionArea != null && highExpressionArea >= 0) {
negativeScore = gradientAreaGap + highExpressionArea / HIGH_EXPRESSION_FACTOR;
Expand All @@ -212,34 +212,32 @@ public static long calculateNegativeScore(Long gradientAreaGap, Long highExpress
}

/**
* The method calculates the normalized score given the gradient area gap the high expression area and the pixel match values
* using the maximum negative score and maximum pixel match for normalization.
* The method calculates the normalized score given the area gap and the pixel match values
* using the maximum area gap and maximum pixel match for normalization.
*
* @param pixelMatch - pixel match size
* @param gradientAreaGap - gradient area gap
* @param highExpressionArea - area of regions with high expression
* @param pixelMatchScore - pixel match size
* @param shapeScore - shape score
* @param maxPixelMatch - maximum pixel size of the current data set.
* @param maxNegativeScore - maximum area gap from the current data set
* @param maxShapeScore - maximum area gap from the current data set
* @return
*/
public static double calculateNormalizedScore(int pixelMatch,
long gradientAreaGap,
long highExpressionArea,
public static double calculateNormalizedScore(int pixelMatchScore,
long shapeScore,
long maxPixelMatch,
long maxNegativeScore) {
if (pixelMatch == 0 || maxPixelMatch == 0 || maxNegativeScore < 0) {
return pixelMatch;
long maxShapeScore) {
if (pixelMatchScore == 0 || maxPixelMatch == 0 || shapeScore < 0 || maxShapeScore <= 0) {
return pixelMatchScore;
} else {
double negativeScore = calculateNegativeScore(gradientAreaGap, highExpressionArea);
if (gradientAreaGap < 0 || maxNegativeScore <= 0 || negativeScore == -1) {
return pixelMatch;
double normalizedShapeScore = (double)shapeScore / maxShapeScore;
double boundedShapeScore;
if (normalizedShapeScore < LOW_NORMALIZED_NEGATIVE_SCORE) {
boundedShapeScore = LOW_NORMALIZED_NEGATIVE_SCORE;
} else if (normalizedShapeScore > HIGH_NORMALIZED_NEGATIVE_SCORE) {
boundedShapeScore = HIGH_NORMALIZED_NEGATIVE_SCORE;
} else {
boundedShapeScore = normalizedShapeScore;
}
double normalizedNegativeScore = negativeScore / maxNegativeScore;
double boundedNegativeScore = Math.min(
Math.max(normalizedNegativeScore * 2.5, LOW_NORMALIZED_NEGATIVE_SCORE),
HIGH_NORMALIZED_NEGATIVE_SCORE
);
return (double)pixelMatch / (double)maxPixelMatch / boundedNegativeScore * 100;
return (double)pixelMatchScore / (double)maxPixelMatch / boundedShapeScore * 100;
}
}

Expand Down
18 changes: 17 additions & 1 deletion colormipsearch-api/src/main/java/org/janelia/colormipsearch/cds/ShapeMatchScore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.apache.commons.lang3.builder.ToStringBuilder;

public class ShapeMatchScore implements ColorDepthMatchScore {
private final long bidirectionalAreaGap;
private final long gradientAreaGap;
private final long highExpressionArea;
private final long maxGradientAreaGap;
Expand All @@ -13,11 +14,22 @@ public class ShapeMatchScore implements ColorDepthMatchScore {
this.highExpressionArea = highExpressionArea;
this.maxGradientAreaGap = maxGradientAreaGap;
this.mirrored = mirrored;
this.bidirectionalAreaGap = -1;
}

ShapeMatchScore(long bidirectionalAreaGap) {
this.bidirectionalAreaGap = bidirectionalAreaGap;
this.gradientAreaGap = -1;
this.highExpressionArea = -1;
this.maxGradientAreaGap = -1;
this.mirrored = false;
}

@Override
public int getScore() {
return (int) GradientAreaGapUtils.calculateNegativeScore(gradientAreaGap, highExpressionArea);
return bidirectionalAreaGap != -1
? (int) bidirectionalAreaGap
: (int) GradientAreaGapUtils.calculate2DShapeScore(gradientAreaGap, highExpressionArea);
}

@Override
Expand All @@ -31,6 +43,10 @@ public boolean isMirrored() {
return mirrored;
}

public long getBidirectionalAreaGap() {
return bidirectionalAreaGap;
}

public long getGradientAreaGap() {
return gradientAreaGap;
}
Expand Down
29 changes: 26 additions & 3 deletions colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/CDMatchEntity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ public class CDMatchEntity<M extends AbstractNeuronEntity, T extends AbstractNeu
private Float normalizedScore;
private Integer matchingPixels;
private Float matchingPixelsRatio;
private Long bidirectionalAreaGap;
private Long gradientAreaGap;
private Long highExpressionArea;
private boolean matchFound;
Expand Down Expand Up @@ -42,6 +43,14 @@ public void setMatchingPixelsRatio(Float matchingPixelsRatio) {
this.matchingPixelsRatio = matchingPixelsRatio;
}

public Long getBidirectionalAreaGap() {
return bidirectionalAreaGap;
}

public void setBidirectionalAreaGap(Long bidirectionalAreaGap) {
this.bidirectionalAreaGap = bidirectionalAreaGap;
}

public Long getGradientAreaGap() {
return gradientAreaGap;
}
Expand All @@ -60,15 +69,28 @@ public void setHighExpressionArea(Long highExpressionArea) {

@JsonIgnore
public Long getGradScore() {
return hasGradScore()
? GradientAreaGapUtils.calculateNegativeScore(gradientAreaGap, highExpressionArea)
: -1;
if (!hasGradScore()) {
return -1L;
}
if (has3DBidirectionalShapeScore()) {
return bidirectionalAreaGap;
} else {
return GradientAreaGapUtils.calculate2DShapeScore(gradientAreaGap, highExpressionArea);
}
}

public boolean hasGradScore() {
return has3DBidirectionalShapeScore() || has2DShapeScore();
}

private boolean has2DShapeScore() {
return gradientAreaGap != null && gradientAreaGap >= 0 && highExpressionArea != null && highExpressionArea > 0;
}

private boolean has3DBidirectionalShapeScore() {
return bidirectionalAreaGap != null && bidirectionalAreaGap >= 0;
}

@JsonIgnore
public boolean isMatchFound() {
return matchFound;
Expand Down Expand Up @@ -106,6 +128,7 @@ public boolean hasNoErrors() {
clone.normalizedScore = this.normalizedScore;
clone.matchingPixels = this.matchingPixels;
clone.matchingPixelsRatio = this.matchingPixelsRatio;
clone.bidirectionalAreaGap = this.bidirectionalAreaGap;
clone.gradientAreaGap = this.gradientAreaGap;
clone.highExpressionArea = this.highExpressionArea;
clone.matchFound = this.matchFound;
Expand Down
2 changes: 2 additions & 0 deletions ...h-persist/src/itest/java/org/janelia/colormipsearch/dao/mongo/CDMatchesMongoDaoITest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ public void findAndUpdate() {
List<Function<CDMatchEntity<EMNeuronEntity, LMNeuronEntity>, EntityField<?>>> fieldsToUpdate = Arrays.asList(
m -> new EntityField<>("matchingPixels", false, m.getMatchingPixels()),
m -> new EntityField<>("matchingPixelsRatio", false, m.getMatchingPixelsRatio()),
m -> new EntityField<>("bidirectionalAreaGap", false, m.getBidirectionalAreaGap()),
m -> new EntityField<>("gradientAreaGap", false, m.getGradientAreaGap()),
m -> new EntityField<>("highExpressionArea", false ,m.getHighExpressionArea()),
m -> new EntityField<>("normalizedScore", false, m.getNormalizedScore()),
Expand Down Expand Up @@ -293,6 +294,7 @@ public void updateExisting() {
List<Function<CDMatchEntity<EMNeuronEntity, LMNeuronEntity>, Pair<String, ?>>> fieldsToUpdate = Arrays.asList(
m -> Pair.of("matchingPixels", m.getMatchingPixels()),
m -> Pair.of("matchingPixelsRatio", m.getMatchingPixelsRatio()),
m -> Pair.of("bidirectionalAreaGap", m.getBidirectionalAreaGap()),
m -> Pair.of("gradientAreaGap", m.getGradientAreaGap()),
m -> Pair.of("highExpressionArea", m.getHighExpressionArea()),
m -> Pair.of("normalizedScore", m.getNormalizedScore())
Expand Down
53 changes: 31 additions & 22 deletions ...search-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@ static class CalculateGradientScoresArgs extends AbstractGradientScoresArgs {
arity = 0)
boolean processPartitionsConcurrently = false;

@Parameter(names = {"--use-bidirectional-matching"},
description = "Use bidirectional matching",
arity = 0)
boolean useBidirectionalMatching = false;

CalculateGradientScoresArgs(CommonArgs commonArgs) {
super(commonArgs);
}
Expand Down Expand Up @@ -184,18 +189,19 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void ca
(Runtime.getRuntime().totalMemory() / _1M));
}

private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void processMasks(List<String> masksIds,
NeuronMatchesReader<CDMatchEntity<M, T>> cdMatchesReader,
NeuronMatchesWriter<CDMatchEntity<M, T>> cdMatchesWriter,
CDMIPsWriter cdmipsWriter,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> gradScoreAlgorithmProvider,
Executor executor,
String processingContext) {
private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity>
void processMasks(List<String> masksIds,
NeuronMatchesReader<CDMatchEntity<M, T>> cdMatchesReader,
NeuronMatchesWriter<CDMatchEntity<M, T>> cdMatchesWriter,
CDMIPsWriter cdmipsWriter,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> shapeScoreAlgorithmProvider,
Executor executor,
String processingContext) {
LOG.info("Start {} - process {} masks", processingContext, masksIds.size());
long startProcessingPartitionTime = System.currentTimeMillis();
long updatedMatches = 0;
for (String maskId : masksIds) {
updatedMatches += processMask(maskId, cdMatchesReader, cdMatchesWriter, cdmipsWriter, gradScoreAlgorithmProvider, executor, processingContext);
updatedMatches += processMask(maskId, cdMatchesReader, cdMatchesWriter, cdmipsWriter, shapeScoreAlgorithmProvider, executor, processingContext);
}
LOG.info("Finished {} - completed {} masks, updated {} matches in {}s - memory usage {}M out of {}M",
processingContext,
Expand All @@ -206,13 +212,14 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void pr
(Runtime.getRuntime().totalMemory() / _1M));
}

private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> long processMask(String maskId,
NeuronMatchesReader<CDMatchEntity<M, T>> cdMatchesReader,
NeuronMatchesWriter<CDMatchEntity<M, T>> cdMatchesWriter,
CDMIPsWriter cdmipsWriter,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> gradScoreAlgorithmProvider,
Executor executor,
String processingContext) {
private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity>
long processMask(String maskId,
NeuronMatchesReader<CDMatchEntity<M, T>> cdMatchesReader,
NeuronMatchesWriter<CDMatchEntity<M, T>> cdMatchesWriter,
CDMIPsWriter cdmipsWriter,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> gradScoreAlgorithmProvider,
Executor executor,
String processingContext) {
// read all matches for the current mask
List<CDMatchEntity<M, T>> cdMatchesForMask = getCDMatchesForMask(cdMatchesReader, maskId);
long nPublishedNames = cdMatchesForMask.stream()
Expand Down Expand Up @@ -315,7 +322,7 @@ private CDMIPsWriter getCDMipsWriter() {
/**
* The method calculates and updates the gradient scores for all color depth matches of the given mask MIP ID.
*
* @param gradScoreAlgorithmProvider grad score algorithm provider
* @param shapeScoreAlgorithmProvider grad score algorithm provider
* @param cdMatches color depth matches for which the grad score will be computed
* @param gradScoreParallelism the degree of parallelism used for calculating the grad score(s)
* @param executor task executor
Expand All @@ -324,7 +331,7 @@ private CDMIPsWriter getCDMipsWriter() {
*/
private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity>
List<CDMatchEntity<M, T>> calculateGradientScores(
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> gradScoreAlgorithmProvider,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> shapeScoreAlgorithmProvider,
List<CDMatchEntity<M, T>> cdMatches,
int gradScoreParallelism,
Executor executor) {
Expand All @@ -342,7 +349,7 @@ List<CDMatchEntity<M, T>> calculateGradientScores(
.flatMap(selectedMaskMatches -> startGradScoreComputations(
selectedMaskMatches.getKey(),
selectedMaskMatches.getItems(),
gradScoreAlgorithmProvider,
shapeScoreAlgorithmProvider,
gradScoreParallelism,
executor
).stream())
Expand All @@ -369,6 +376,7 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> long up
return cdMatchesWriter.writeUpdates(
cdMatches,
Arrays.asList(
m -> ImmutablePair.of("bidirectionalAreaGap", m.getBidirectionalAreaGap()),
m -> ImmutablePair.of("gradientAreaGap", m.getGradientAreaGap()),
m -> ImmutablePair.of("highExpressionArea", m.getHighExpressionArea()),
m -> ImmutablePair.of("normalizedScore", m.getNormalizedScore()),
Expand Down Expand Up @@ -436,7 +444,7 @@ List<CDMatchEntity<M, T>> getCDMatchesForMask(NeuronMatchesReader<CDMatchEntity<
private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity>
List<CompletableFuture<List<CDMatchEntity<M, T>>>> startGradScoreComputations(M mask,
List<CDMatchEntity<M, T>> selectedMatches,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> gradScoreAlgorithmProvider,
ColorDepthSearchAlgorithmProvider<ShapeMatchScore> shapeScoreAlgorithmProvider,
int gradScoreParallelism,
Executor executor) {
if (CollectionUtils.isEmpty(selectedMatches)) {
Expand All @@ -451,7 +459,7 @@ List<CompletableFuture<List<CDMatchEntity<M, T>>>> startGradScoreComputations(M
return Collections.emptyList();
}
ColorDepthSearchAlgorithm<ShapeMatchScore> gradScoreAlgorithm =
gradScoreAlgorithmProvider.createColorDepthQuerySearchAlgorithmWithDefaultParams(
shapeScoreAlgorithmProvider.createColorDepthQuerySearchAlgorithmWithDefaultParams(
maskImage.getImageArray(),
args.maskThreshold,
args.borderSize);
Expand Down Expand Up @@ -482,12 +490,14 @@ List<CompletableFuture<List<CDMatchEntity<M, T>>>> startGradScoreComputations(M
(n, cft) -> NeuronMIPUtils.getImageArray(CachedMIPsUtils.loadMIP(n, cft))
)
);
cdsMatch.setBidirectionalAreaGap(gradScore.getBidirectionalAreaGap());
cdsMatch.setGradientAreaGap(gradScore.getGradientAreaGap());
cdsMatch.setHighExpressionArea(gradScore.getHighExpressionArea());
cdsMatch.setNormalizedScore(gradScore.getNormalizedScore());
LOG.debug("Finished calculating negative score between {} and {} in {}ms",
cdsMatch.getMaskImage(), cdsMatch.getMatchedImage(), System.currentTimeMillis() - startCalcTime);
} else {
cdsMatch.setBidirectionalAreaGap(-1L);
cdsMatch.setGradientAreaGap(-1L);
cdsMatch.setHighExpressionArea(-1L);
}
Expand All @@ -509,8 +519,7 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void up
// update normalized score
cdMatches.forEach(m -> m.setNormalizedScore((float) GradientAreaGapUtils.calculateNormalizedScore(
m.getMatchingPixels(),
m.getGradientAreaGap(),
m.getHighExpressionArea(),
m.getGradScore(),
maxScores.getPixelMatches(),
maxScores.getGradScore()
)));
Expand Down
Loading

0 comments on commit 06fcabf

Please sign in to comment.