Skip to content

Commit

Permalink
Optimized DLS/FLS
Browse files Browse the repository at this point in the history 
Signed-off-by: Nils Bandener <[email protected]>
  • Loading branch information
@nibix
nibix committed Sep 11, 2024
1 parent e8ca33f commit 31cb1c6
Show file tree
Hide file tree
Showing 28 changed files with 4,587 additions and 1,111 deletions.
1,199 changes: 1,199 additions & 0 deletions ...ntegrationTest/java/org/opensearch/security/privileges/dlsfls/DocumentPrivilegesTest.java
View file

Large diffs are not rendered by default.

24 changes: 24 additions & 0 deletions src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -54,8 +55,10 @@
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.Strings;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.security.hasher.PasswordHasher;
import org.opensearch.security.hasher.PasswordHasherFactory;
import org.opensearch.security.securityconf.impl.CType;
Expand Down Expand Up @@ -651,6 +654,22 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(name, clusterPermissions, indexPermissions, hidden, reserved, description);
}

public static SecurityDynamicConfiguration<org.opensearch.security.securityconf.impl.v7.RoleV7> toRolesConfiguration(
TestSecurityConfig.Role... roles
) {
try {
return SecurityDynamicConfiguration.fromJson(
configToJson(CType.ROLES, Stream.of(roles).collect(Collectors.toMap(r -> r.name, r -> r))),
CType.ROLES,
2,
0,
0
);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

public static class RoleMapping implements ToXContentObject {
Expand Down Expand Up @@ -764,6 +783,11 @@ public IndexPermission dls(String dlsQuery) {
return this;
}

public IndexPermission dls(QueryBuilder dlsQuery) {
this.dlsQuery = Strings.toString(MediaTypeRegistry.JSON, dlsQuery);
return this;
}

public IndexPermission fls(String... fls) {
this.fls = Arrays.asList(fls);
return this;
Expand Down
129 changes: 44 additions & 85 deletions src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -153,7 +152,6 @@
import org.opensearch.security.configuration.DlsFlsRequestValve;
import org.opensearch.security.configuration.DlsFlsValveImpl;
import org.opensearch.security.configuration.PrivilegesInterceptorImpl;
import org.opensearch.security.configuration.Salt;
import org.opensearch.security.configuration.SecurityFlsDlsIndexSearcherWrapper;
import org.opensearch.security.dlic.rest.api.Endpoint;
import org.opensearch.security.dlic.rest.api.SecurityRestApiActions;
Expand All @@ -169,9 +167,11 @@
import org.opensearch.security.identity.NoopPluginSubject;
import org.opensearch.security.identity.SecurityTokenManager;
import org.opensearch.security.privileges.ActionPrivileges;
import org.opensearch.security.privileges.PrivilegesEvaluationException;
import org.opensearch.security.privileges.PrivilegesEvaluator;
import org.opensearch.security.privileges.PrivilegesInterceptor;
import org.opensearch.security.privileges.RestLayerPrivilegesEvaluator;
import org.opensearch.security.privileges.dlsfls.DlsFlsBaseContext;
import org.opensearch.security.resolver.IndexResolverReplacer;
import org.opensearch.security.rest.DashboardsInfoAction;
import org.opensearch.security.rest.SecurityConfigUpdateAction;
Expand All @@ -180,6 +180,7 @@
import org.opensearch.security.rest.SecurityWhoAmIAction;
import org.opensearch.security.rest.TenantInfoAction;
import org.opensearch.security.securityconf.DynamicConfigFactory;
import org.opensearch.security.securityconf.impl.CType;
import org.opensearch.security.setting.OpensearchDynamicSetting;
import org.opensearch.security.setting.TransportPassiveAuthSetting;
import org.opensearch.security.ssl.ExternalSecurityKeyStore;
Expand All @@ -196,8 +197,6 @@
import org.opensearch.security.support.ModuleInfo;
import org.opensearch.security.support.ReflectionHelper;
import org.opensearch.security.support.SecuritySettings;
import org.opensearch.security.support.SecurityUtils;
import org.opensearch.security.support.WildcardMatcher;
import org.opensearch.security.transport.DefaultInterClusterRequestEvaluator;
import org.opensearch.security.transport.InterClusterRequestEvaluator;
import org.opensearch.security.transport.SecurityInterceptor;
Expand Down Expand Up @@ -266,9 +265,9 @@ public final class OpenSearchSecurityPlugin extends OpenSearchSecuritySSLPlugin
private volatile IndexResolverReplacer irr;
private final AtomicReference<NamedXContentRegistry> namedXContentRegistry = new AtomicReference<>(NamedXContentRegistry.EMPTY);;
private volatile DlsFlsRequestValve dlsFlsValve = null;
private volatile Salt salt;
private volatile OpensearchDynamicSetting<Boolean> transportPassiveAuthSetting;
private volatile PasswordHasher passwordHasher;
private volatile DlsFlsBaseContext dlsFlsBaseContext;

public static boolean isActionTraceEnabled() {

Expand Down Expand Up @@ -705,7 +704,8 @@ public void onIndexModule(IndexModule indexModule) {
auditLog,
ciol,
evaluator,
salt
dlsFlsValve::getCurrentConfig,
dlsFlsBaseContext
)
);
indexModule.forceQueryCacheProvider((indexSettings, nodeCache) -> new QueryCache() {
Expand All @@ -727,28 +727,18 @@ public void clear(String reason) {

@Override
public Weight doCache(Weight weight, QueryCachingPolicy policy) {
@SuppressWarnings("unchecked")
final Map<String, Set<String>> allowedFlsFields = (Map<String, Set<String>>) HeaderHelper.deserializeSafeFromHeader(
threadPool.getThreadContext(),
ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER
);

if (SecurityUtils.evalMap(allowedFlsFields, index().getName()) != null) {
return weight;
} else {
@SuppressWarnings("unchecked")
final Map<String, Set<String>> maskedFieldsMap = (Map<String, Set<String>>) HeaderHelper.deserializeSafeFromHeader(
threadPool.getThreadContext(),
ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER
);

if (SecurityUtils.evalMap(maskedFieldsMap, index().getName()) != null) {
try {
if (dlsFlsValve.hasFlsOrFieldMasking(index().getName())) {
// Do not cache
return weight;
} else {
return nodeCache.doCache(weight, policy);
}
} catch (PrivilegesEvaluationException e) {
log.error("Error while evaluating FLS configuration", e);
// We fall back to no caching
return weight;
}

}
});

Expand Down Expand Up @@ -821,17 +811,14 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
return;
}

@SuppressWarnings("unchecked")
final Map<String, Set<String>> maskedFieldsMap = (Map<String, Set<String>>) HeaderHelper.deserializeSafeFromHeader(
threadPool.getThreadContext(),
ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER
);
final String maskedEval = SecurityUtils.evalMap(maskedFieldsMap, indexModule.getIndex().getName());
if (maskedEval != null) {
final Set<String> mf = maskedFieldsMap.get(maskedEval);
if (mf != null && !mf.isEmpty()) {
try {
if (dlsFlsValve.hasFieldMasking(indexModule.getIndex().getName())) {
dlsFlsValve.onQueryPhase(queryResult);
}
} catch (PrivilegesEvaluationException e) {
log.error("Error while evaluating field masking config", e);
// It is safe to call the code nevertheless
dlsFlsValve.onQueryPhase(queryResult);
}
}
}.toListener());
Expand Down Expand Up @@ -1065,7 +1052,6 @@ public Collection<Object> createComponents(

final ClusterInfoHolder cih = new ClusterInfoHolder(this.cs.getClusterName().value());
this.cs.addListener(cih);
this.salt = Salt.from(settings);

final IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(threadPool.getThreadContext());
irr = new IndexResolverReplacer(resolver, clusterService::state, cih);
Expand All @@ -1086,18 +1072,9 @@ public Collection<Object> createComponents(

namedXContentRegistry.set(xContentRegistry);
if (SSLConfig.isSslOnlyMode()) {
dlsFlsValve = new DlsFlsRequestValve.NoopDlsFlsRequestValve();
auditLog = new NullAuditLog();
privilegesInterceptor = new PrivilegesInterceptor(resolver, clusterService, localClient, threadPool);
} else {
dlsFlsValve = new DlsFlsValveImpl(
settings,
localClient,
clusterService,
resolver,
xContentRegistry,
threadPool.getThreadContext()
);
auditLog = new AuditLogImpl(settings, configPath, localClient, threadPool, resolver, clusterService, environment);
privilegesInterceptor = new PrivilegesInterceptorImpl(resolver, clusterService, localClient, threadPool);
}
Expand Down Expand Up @@ -1133,6 +1110,23 @@ public Collection<Object> createComponents(
namedXContentRegistry.get()
);

dlsFlsBaseContext = new DlsFlsBaseContext(evaluator, threadPool.getThreadContext());

if (SSLConfig.isSslOnlyMode()) {
dlsFlsValve = new DlsFlsRequestValve.NoopDlsFlsRequestValve();
} else {
dlsFlsValve = new DlsFlsValveImpl(
settings,
localClient,
clusterService,
resolver,
xContentRegistry,
threadPool,
dlsFlsBaseContext
);
cr.subscribeOnChange(configMap -> { ((DlsFlsValveImpl) dlsFlsValve).updateConfiguration(cr.getConfiguration(CType.ROLES)); });
}

sf = new SecurityFilter(settings, evaluator, adminDns, dlsFlsValve, auditLog, threadPool, cs, compatConfig, irr, xffResolver);

final String principalExtractorClass = settings.get(SSLConfigConstants.SECURITY_SSL_TRANSPORT_PRINCIPAL_EXTRACTOR_CLASS, null);
Expand Down Expand Up @@ -1167,9 +1161,6 @@ public Collection<Object> createComponents(
// Don't register if advanced modules is disabled in which case auditlog is instance of NullAuditLog
dcf.registerDCFListener(auditLog);
}
if (dlsFlsValve instanceof DlsFlsValveImpl) {
dcf.registerDCFListener(dlsFlsValve);
}

cr.setDynamicConfigFactory(dcf);

Expand Down Expand Up @@ -2053,43 +2044,18 @@ public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses()
@Override
public Function<String, Predicate<String>> getFieldFilter() {
return index -> {
if (threadPool == null) {
if (threadPool == null || dlsFlsValve == null) {
return field -> true;
}
@SuppressWarnings("unchecked")
final Map<String, Set<String>> allowedFlsFields = (Map<String, Set<String>>) HeaderHelper.deserializeSafeFromHeader(
threadPool.getThreadContext(),
ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER
);

final String eval = SecurityUtils.evalMap(allowedFlsFields, index);

if (eval == null) {
return field -> true;
} else {

final Set<String> includesExcludes = allowedFlsFields.get(eval);
final Set<String> includesSet = new HashSet<>(includesExcludes.size());
final Set<String> excludesSet = new HashSet<>(includesExcludes.size());

for (final String incExc : includesExcludes) {
final char firstChar = incExc.charAt(0);

if (firstChar == '!' || firstChar == '~') {
excludesSet.add(incExc.substring(1));
} else {
includesSet.add(incExc);
}
}

if (!excludesSet.isEmpty()) {
WildcardMatcher excludeMatcher = WildcardMatcher.from(excludesSet);
return field -> !excludeMatcher.test(handleKeyword(field));
} else {
WildcardMatcher includeMatcher = WildcardMatcher.from(includesSet);
return field -> includeMatcher.test(handleKeyword(field));
return field -> {
try {
return dlsFlsValve.isFieldAllowed(index, field);
} catch (PrivilegesEvaluationException e) {
log.error("Error while evaluating FLS for {}.{}", index, field, e);
return false;
}
}
};
};
}

Expand All @@ -2103,13 +2069,6 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
return Collections.singletonList(systemIndexDescriptor);
}

private static String handleKeyword(final String field) {
if (field != null && field.endsWith(KEYWORD)) {
return field.substring(0, field.length() - KEYWORD.length());
}
return field;
}

@Override
public Subject getCurrentSubject() {
// Not supported
Expand Down
11 changes: 6 additions & 5 deletions src/main/java/org/opensearch/security/compliance/FieldReadCallback.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@
import org.opensearch.index.mapper.Uid;
import org.opensearch.security.auditlog.AuditLog;
import org.opensearch.security.dlic.rest.support.Utils;
import org.opensearch.security.privileges.dlsfls.FieldMasking;
import org.opensearch.security.support.HeaderHelper;
import org.opensearch.security.support.JsonFlattener;
import org.opensearch.security.support.SourceFieldsContext;
import org.opensearch.security.support.WildcardMatcher;

//TODO We need to deal with caching!!
//Currently we disable caching (and realtime requests) when FLS or DLS is applied
Expand All @@ -49,7 +49,7 @@ public final class FieldReadCallback {
// private final ThreadContext threadContext;
// private final ClusterService clusterService;
private final Index index;
private final WildcardMatcher maskedFieldsMatcher;
private final FieldMasking.FieldMaskingRule fmRule;
private final AuditLog auditLog;
private Function<Map<String, ?>, Map<String, Object>> filterFunction;
private SourceFieldsContext sfc;
Expand All @@ -61,15 +61,15 @@ public FieldReadCallback(
final IndexService indexService,
final ClusterService clusterService,
final AuditLog auditLog,
final WildcardMatcher maskedFieldsMatcher,
final FieldMasking.FieldMaskingRule fmRule,
ShardId shardId
) {
super();
// this.threadContext = Objects.requireNonNull(threadContext);
// this.clusterService = Objects.requireNonNull(clusterService);
this.index = Objects.requireNonNull(indexService).index();
this.auditLog = auditLog;
this.maskedFieldsMatcher = maskedFieldsMatcher;
this.fmRule = fmRule;
this.shardId = shardId;
try {
sfc = (SourceFieldsContext) HeaderHelper.deserializeSafeFromHeader(threadContext, "_opendistro_security_source_field_context");
Expand All @@ -88,7 +88,8 @@ public FieldReadCallback(
}

private boolean recordField(final String fieldName, boolean isStringField) {
return !(isStringField && maskedFieldsMatcher.test(fieldName))
// We do not record fields in read history if they are masked.
return !(isStringField && fmRule.isMasked(fieldName))
&& auditLog.getComplianceConfig().readHistoryEnabledForField(index.getName(), fieldName);
}

Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ public void singleFailure(Failure failure) {
"Failure {} retrieving configuration for {} (index={})",
failure == null ? null : failure.getMessage(),
Arrays.toString(events),
securityIndex
securityIndex,
failure.getFailure()
);
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ public synchronized void subscribeOnChange(ConfigurationChangeListener listener)
private synchronized void notifyAboutChanges(Map<CType, SecurityDynamicConfiguration<?>> typeToConfig) {
for (ConfigurationChangeListener listener : configurationChangedListener) {
try {
LOGGER.debug("Notify {} listener about change configuration with type {}", listener);
LOGGER.debug("Notify {} listener about change configuration with type {}", listener, typeToConfig);
listener.onChange(typeToConfig);
} catch (Exception e) {
LOGGER.error("{} listener errored: " + e, listener, e);
Expand Down
Loading

0 comments on commit 31cb1c6

Please sign in to comment.