Skip to content

Commit

Permalink
Merge pull request #12 from eirsep/2.13-threat-intel
Browse files Browse the repository at this point in the history 
fix update threat intel monitor to avoid monitor exists check before …
  • Loading branch information
@AWSHurneyt
AWSHurneyt authored Jul 18, 2024
2 parents 53486a2 + e44a76c commit 0b34c61
Show file tree
Hide file tree
Showing 6 changed files with 483 additions and 49 deletions.
11 changes: 9 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import org.opensearch.commons.alerting.model.Monitor;
import org.opensearch.commons.alerting.model.MonitorMetadata;
import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput;
import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger;

import java.util.List;
import java.util.Map;
Expand All @@ -16,14 +15,18 @@ public class IocScanContext<Data> {
private final ThreatIntelInput threatIntelInput; // deserialize threat intel input
private final List<String> indices; // user's log data indices
private final Map<String, List<String>> iocTypeToIndices;
public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List<Data> data, ThreatIntelInput threatIntelInput, List<String> indices, Map<String, List<String>> iocTypeToIndices) {
private final Map<String, List<String>> concreteIndexToMonitorInputIndicesMap;

public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List<Data> data, ThreatIntelInput threatIntelInput, List<String> indices, Map<String, List<String>> iocTypeToIndices, Map<String,
List<String>> concreteIndexToMonitorInputIndicesMap) {
this.monitor = monitor;
this.monitorMetadata = monitorMetadata;
this.dryRun = dryRun;
this.data = data;
this.threatIntelInput = threatIntelInput;
this.indices = indices;
this.iocTypeToIndices = iocTypeToIndices;
this.concreteIndexToMonitorInputIndicesMap = concreteIndexToMonitorInputIndicesMap;
}

public Monitor getMonitor() {
Expand All @@ -50,6 +53,10 @@ public List<String> getIndices() {
return indices;
}

public Map<String, List<String>> getConcreteIndexToMonitorInputIndicesMap() {
return concreteIndexToMonitorInputIndicesMap;
}

public Map<String, List<String>> getIocTypeToIndices() {
return iocTypeToIndices;
}
Expand Down
46 changes: 26 additions & 20 deletions ...ain/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void scanIoCs(IocScanContext<Data> iocScanContext,
Monitor monitor = iocScanContext.getMonitor();

long startTime = System.currentTimeMillis();
IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext.getThreatIntelInput().getPerIocTypeScanInputList());
IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext);
BiConsumer<List<STIX2IOC>, Exception> iocScanResultConsumer = (List<STIX2IOC> maliciousIocs, Exception e) -> {
long scanEndTime = System.currentTimeMillis();
long timeTaken = scanEndTime - startTime;
Expand Down Expand Up @@ -113,31 +113,37 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
* 4. doc id to iocs map (reverse mapping of 2)
*/
private IocLookupDtos extractIocsPerType
(List<Data> data, List<PerIocTypeScanInput> iocTypeToIndexFieldMappings) {
(List<Data> data, IocScanContext<Data> context) {
Map<String, Set<String>> iocsPerIocTypeMap = new HashMap<>();
Map<String, Set<String>> iocValueToDocIdMap = new HashMap<>();
Map<String, Set<String>> docIdToIocsMap = new HashMap<>();
for (Data datum : data) {
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) {
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) {
String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase();
String index = getIndexName(datum);
List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
for (String field : fields) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
iocs.addAll(vals);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
String concreteIndex = getIndexName(datum);
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)
&& false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty()
) {
// if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc.
String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0);
List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
for (String field : fields) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
}
}
}
}
Expand Down
15 changes: 13 additions & 2 deletions .../securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.opensearch.action.support.GroupedActionListener;
import org.opensearch.action.support.HandledTransportAction;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Settings;
Expand All @@ -34,6 +35,7 @@
import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService;
import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
import org.opensearch.securityanalytics.util.IndexUtils;
import org.opensearch.tasks.Task;
import org.opensearch.transport.TransportService;

Expand All @@ -48,6 +50,7 @@
import java.util.function.BiConsumer;

import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelInputFromBytesReference;
import static org.opensearch.securityanalytics.util.IndexUtils.getConcreteindexToMonitorInputIndicesMap;

public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAction<DocLevelMonitorFanOutRequest, DocLevelMonitorFanOutResponse> {
private static final Logger log = LogManager.getLogger(TransportThreatIntelMonitorFanOutAction.class);
Expand All @@ -60,6 +63,7 @@ public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAct

private final NamedXContentRegistry xContentRegistry;
private final SaIoCScanService saIoCScanService;
private final IndexNameExpressionResolver indexNameExpressionResolver;

@Inject
public TransportThreatIntelMonitorFanOutAction(
Expand All @@ -70,7 +74,8 @@ public TransportThreatIntelMonitorFanOutAction(
Settings settings,
ActionFilters actionFilters,
SATIFSourceConfigService saTifSourceConfigService,
SaIoCScanService saIoCScanService
SaIoCScanService saIoCScanService,
IndexNameExpressionResolver indexNameExpressionResolver
) {
super(ThreatIntelMonitorRunner.FAN_OUT_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new);
this.clusterService = clusterService;
Expand All @@ -79,6 +84,7 @@ public TransportThreatIntelMonitorFanOutAction(
this.settings = settings;
this.saTifSourceConfigService = saTifSourceConfigService;
this.saIoCScanService = saIoCScanService;
this.indexNameExpressionResolver = indexNameExpressionResolver;
}

@Override
Expand Down Expand Up @@ -173,14 +179,19 @@ private void onGetIocTypeToIndices(Map<String, List<String>> iocTypeToIndicesMap
actionListener.onFailure(e);
}
};
Map<String, List<String>> concreteindexToMonitorInputIndicesMap = getConcreteindexToMonitorInputIndicesMap(
remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(),
clusterService,
indexNameExpressionResolver);
saIoCScanService.scanIoCs(new IocScanContext<>(
request.getMonitor(),
request.getMonitorMetadata(),
false,
hits,
threatIntelInput,
indices,
iocTypeToIndicesMap
iocTypeToIndicesMap,
concreteindexToMonitorInputIndicesMap
), resultConsumer);
},
e -> {
Expand Down
2 changes: 1 addition & 1 deletion ...n/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,7 @@ public void getIocTypeToIndices(ActionListener<Map<String, List<String>>> listen
String stateFieldName = getStateFieldName();
BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery()
.should(QueryBuilders.matchQuery(stateFieldName, AVAILABLE.toString()));
stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING));
stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING.toString()));
queryBuilder.must(stateQueryBuilder);

searchRequest.source().query(queryBuilder);
Expand Down
33 changes: 27 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,6 @@
*/
package org.opensearch.securityanalytics.util;

import java.util.Optional;
import java.util.SortedMap;

import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.core.action.ActionListener;
import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.action.support.master.AcknowledgedResponse;
Expand All @@ -17,17 +12,22 @@
import org.opensearch.cluster.metadata.IndexAbstraction;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;
import java.util.Optional;
import java.util.SortedMap;

public class IndexUtils {

Expand Down Expand Up @@ -210,4 +210,25 @@ public static String getIndexNameWithAlias(ClusterState clusterState, String ali
return entry.map(Map.Entry::getKey).orElse(null);
}

public static Map<String, List<String>> getConcreteindexToMonitorInputIndicesMap(List<String> indices, ClusterService clusterService, IndexNameExpressionResolver resolver) {
Map<String, List<String>> result = new HashMap<>();

for (String index : indices) {
String[] concreteIndices = resolver.concreteIndexNames(
clusterService.state(),
IndicesOptions.lenientExpand(),
true,
index
);
for (String concreteIndex : concreteIndices) {
if (!result.containsKey(concreteIndex)) {
result.put(concreteIndex, new ArrayList<>());
}
result.get(concreteIndex).add(index);
}
}

return result;
}

}
Loading

0 comments on commit 0b34c61

Please sign in to comment.