Skip to content

Commit

Permalink
DLS/FLS backward compat
Browse files Browse the repository at this point in the history 
Signed-off-by: Nils Bandener <[email protected]>
  • Loading branch information
@nibix
nibix committed Oct 16, 2024
1 parent 73bf5de commit 5114d76
Show file tree
Hide file tree
Showing 6 changed files with 293 additions and 200 deletions.
72 changes: 68 additions & 4 deletions ...tegrationTest/java/org/opensearch/security/privileges/dlsfls/DlsFlsLegacyHeadersTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/
package org.opensearch.security.privileges.dlsfls;

import java.io.IOException;
Expand Down Expand Up @@ -32,6 +42,7 @@
import static org.opensearch.security.Song.FIELD_ARTIST;
import static org.opensearch.security.Song.FIELD_STARS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;

public class DlsFlsLegacyHeadersTest {
static NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(
Expand Down Expand Up @@ -73,7 +84,8 @@ public void dls_simple() throws Exception {
String header = new DlsFlsLegacyHeaders(
ctx(metadata, "read_where_field_artist_matches_artist_string"),
dlsFlsProcessedConfig,
metadata
metadata,
false
).getDlsHeader();

// Created with DlsIntegrationTests.testShouldSearchI1_S2I2_S3() on an earlier OpenSearch version
Expand Down Expand Up @@ -106,7 +118,8 @@ public void dls_twoRoles() throws Exception {
String header = new DlsFlsLegacyHeaders(
ctx(metadata, "read_where_field_artist_matches_artist_twins", "read_where_field_stars_greater_than_five"),
dlsFlsProcessedConfig,
metadata
metadata,
false
).getDlsHeader();

// Created with DlsIntegrationTests.testShouldSearchI1_S3I1_S6I2_S2() on an earlier OpenSearch version
Expand All @@ -116,6 +129,19 @@ public void dls_twoRoles() throws Exception {
assertEquals(Base64Helper.deserializeObject(expectedHeader), Base64Helper.deserializeObject(header));
}

@Test
public void dls_none() throws Exception {
SecurityDynamicConfiguration<RoleV7> rolesConfig = TestSecurityConfig.Role.toRolesConfiguration(
new TestSecurityConfig.Role("role").clusterPermissions("cluster_composite_ops_ro").indexPermissions("read").on("*")
);

Metadata metadata = MockIndexMetadataBuilder.indices("first-test-index", "second-test-index", "my_index1").build();
DlsFlsProcessedConfig dlsFlsProcessedConfig = dlsFlsProcessedConfig(rolesConfig, metadata);
String header = new DlsFlsLegacyHeaders(ctx(metadata, "role"), dlsFlsProcessedConfig, metadata, false).getDlsHeader();

assertNull(header);
}

/**
* Basic test that the FLS header matches the one produced in previous versions.
* <p>
Expand All @@ -132,7 +158,8 @@ public void fls_simple() throws Exception {

Metadata metadata = MockIndexMetadataBuilder.indices("first-test-index", "second-test-index", "fls_index").build();
DlsFlsProcessedConfig dlsFlsProcessedConfig = dlsFlsProcessedConfig(rolesConfig, metadata);
String header = new DlsFlsLegacyHeaders(ctx(metadata, "fls_exclude_stars_reader"), dlsFlsProcessedConfig, metadata).getFlsHeader();
String header = new DlsFlsLegacyHeaders(ctx(metadata, "fls_exclude_stars_reader"), dlsFlsProcessedConfig, metadata, false)
.getFlsHeader();

// Created with FlsAndFieldMaskingTests.flsEnabledFieldsAreHiddenForNormalUsers() on an earlier OpenSearch version
String expectedHeader =
Expand All @@ -141,6 +168,42 @@ public void fls_simple() throws Exception {
assertEquals(Base64Helper.deserializeObject(expectedHeader), Base64Helper.deserializeObject(header));
}

/**
* Test that the FLS header matches the one produced in previous versions. In this case, inclusion and exclusion is mixed
* and contradicts itself.
* <p>
* Test configuration corresponds to FlsAndFieldMaskingTests.testGetDocumentWithNoTitleFieldAndOnlyTitleFieldFLSRestrictions()
*/
@Test
public void fls_mixedContradiction() throws Exception {
SecurityDynamicConfiguration<RoleV7> rolesConfig = TestSecurityConfig.Role.toRolesConfiguration(
new TestSecurityConfig.Role("example_inclusive_fls").clusterPermissions("cluster_composite_ops_ro")
.indexPermissions("read")
.fls("title")
.on("first-test-index"),
new TestSecurityConfig.Role("example_exclusive_fls").clusterPermissions("cluster_composite_ops_ro")
.indexPermissions("read")
.fls(String.format("~title"))
.on("first-test-index")
);

Metadata metadata = MockIndexMetadataBuilder.indices("first-test-index", "second-test-index", "fls_index").build();
DlsFlsProcessedConfig dlsFlsProcessedConfig = dlsFlsProcessedConfig(rolesConfig, metadata);
String header = new DlsFlsLegacyHeaders(
ctx(metadata, "example_inclusive_fls", "example_exclusive_fls"),
dlsFlsProcessedConfig,
metadata,
false
).getFlsHeader();

// Created with FlsAndFieldMaskingTests.testGetDocumentWithNoTitleFieldAndOnlyTitleFieldFLSRestrictions() on an earlier OpenSearch
// version
String expectedHeader =
"rO0ABXNyACVqYXZhLnV0aWwuQ29sbGVjdGlvbnMkVW5tb2RpZmlhYmxlTWFw8aWo/nT1B0ICAAFMAAFtdAAPTGphdmEvdXRpbC9NYXA7eHBzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQAEGZpcnN0LXRlc3QtaW5kZXhzcgARamF2YS51dGlsLkhhc2hTZXS6RIWVlri3NAMAAHhwdwwAAAAQP0AAAAAAAAJ0AAV0aXRsZXQABn50aXRsZXh4";

assertEquals(Base64Helper.deserializeObject(expectedHeader), Base64Helper.deserializeObject(header));
}

/**
* Basic test that the field masking header matches the one produced in previous versions.
* <p>
Expand All @@ -164,7 +227,8 @@ public void fieldMasking_simple() throws Exception {
String header = new DlsFlsLegacyHeaders(
ctx(metadata, "masked_title_artist_lyrics_reader", "masked_lyrics_reader"),
dlsFlsProcessedConfig,
metadata
metadata,
false
).getFmHeader();

// Created with FlsAndFieldMaskingTests.flsEnabledFieldsAreHiddenForNormalUsers() on an earlier OpenSearch version
Expand Down
178 changes: 3 additions & 175 deletions src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,17 @@

package org.opensearch.security.configuration;

import java.io.Serializable;
import java.lang.reflect.Field;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.StreamSupport;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.BooleanClause.Occur;
Expand All @@ -37,10 +32,8 @@
import org.opensearch.OpenSearchException;
import org.opensearch.OpenSearchSecurityException;
import org.opensearch.SpecialPermission;
import org.opensearch.Version;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.RealtimeRequest;
import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
import org.opensearch.action.admin.indices.shrink.ResizeRequest;
import org.opensearch.action.bulk.BulkItemRequest;
import org.opensearch.action.bulk.BulkShardRequest;
Expand Down Expand Up @@ -78,18 +71,16 @@
import org.opensearch.security.privileges.PrivilegesEvaluationContext;
import org.opensearch.security.privileges.PrivilegesEvaluationException;
import org.opensearch.security.privileges.dlsfls.DlsFlsBaseContext;
import org.opensearch.security.privileges.dlsfls.DlsFlsLegacyHeaders;
import org.opensearch.security.privileges.dlsfls.DlsFlsProcessedConfig;
import org.opensearch.security.privileges.dlsfls.DlsRestriction;
import org.opensearch.security.privileges.dlsfls.FieldMasking;
import org.opensearch.security.privileges.dlsfls.FieldPrivileges;
import org.opensearch.security.privileges.dlsfls.IndexToRuleMap;
import org.opensearch.security.resolver.IndexResolverReplacer;
import org.opensearch.security.securityconf.DynamicConfigFactory;
import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration;
import org.opensearch.security.securityconf.impl.v7.RoleV7;
import org.opensearch.security.support.Base64Helper;
import org.opensearch.security.support.ConfigConstants;
import org.opensearch.security.support.HeaderHelper;
import org.opensearch.threadpool.ThreadPool;

public class DlsFlsValveImpl implements DlsFlsRequestValve {
Expand Down Expand Up @@ -199,18 +190,8 @@ public boolean invoke(PrivilegesEvaluationContext context, final ActionListener<
}
}

if (legacyHeadersRequired()) {
Set<String> indices = clusterService.state().metadata().indices().keySet();

if (!doFilterLevelDls) {
setDlsHeaders(config.getDocumentPrivileges().getRestrictions(context, indices), request);
}

setFlsHeaders(
config.getFieldPrivileges().getRestrictions(context, indices),
config.getFieldMasking().getRestrictions(context, indices),
request
);
if (DlsFlsLegacyHeaders.possiblyRequired(clusterService)) {
DlsFlsLegacyHeaders.prepare(threadContext, context, config, clusterService.state().metadata(), doFilterLevelDls);
}

if (request instanceof RealtimeRequest) {
Expand Down Expand Up @@ -523,58 +504,6 @@ private static List<StringTerms.Bucket> mergeBuckets(
return buckets;
}

private boolean legacyHeadersRequired() {
// TODO this needs to be adapted if backported
return !clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_3_0_0);
}

private void setDlsHeaders(IndexToRuleMap<DlsRestriction> dlsRestrictionMap, ActionRequest request) {
if (!dlsRestrictionMap.getIndexMap().isEmpty()) {
Map<String, Set<String>> dlsQueriesByIndex = new HashMap<>();

for (Map.Entry<String, DlsRestriction> entry : dlsRestrictionMap.getIndexMap().entrySet()) {
/* dlsQueriesByIndex.put(
entry.getKey(),
entry.getValue()
.getQueries()
.stream()
.map(queryBuilder -> Strings.toString(MediaTypeRegistry.JSON, queryBuilder))
.collect(Collectors.toSet())
);*/
}

if (request instanceof ClusterSearchShardsRequest && HeaderHelper.isTrustedClusterRequest(threadContext)) {
threadContext.addResponseHeader(
ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER,
Base64Helper.serializeObject((Serializable) dlsQueriesByIndex)
);
if (log.isDebugEnabled()) {
log.debug("added response header for DLS info: {}", dlsQueriesByIndex);
}
} else {
if (threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER) != null) {
Object deserializedDlsQueries = Base64Helper.deserializeObject(
threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER),
threadContext.getTransient(ConfigConstants.USE_JDK_SERIALIZATION)
);
if (!dlsQueriesByIndex.equals(deserializedDlsQueries)) {
throw new OpenSearchSecurityException(
ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER + " does not match (SG 900D)"
);
}
} else {
threadContext.putHeader(
ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER,
Base64Helper.serializeObject((Serializable) dlsQueriesByIndex)
);
if (log.isDebugEnabled()) {
log.debug("attach DLS info: {}", dlsQueriesByIndex);
}
}
}
}
}

private void setDlsModeHeader(Mode mode) {
String modeString = mode.name();

Expand Down Expand Up @@ -602,107 +531,6 @@ private Mode getDlsModeHeader() {
}
}

private void setFlsHeaders(
IndexToRuleMap<FieldPrivileges.FlsRule> flsRuleMap,
IndexToRuleMap<FieldMasking.FieldMaskingRule> fmRuleMap,
ActionRequest request
) {
if (!fmRuleMap.isUnrestricted()) {
Map<String, Set<String>> maskedFieldsMap = new HashMap<>();

for (Map.Entry<String, FieldMasking.FieldMaskingRule> entry : fmRuleMap.getIndexMap().entrySet()) {
maskedFieldsMap.put(entry.getKey(), Sets.newHashSet(entry.getValue().getSource()));
}

if (request instanceof ClusterSearchShardsRequest && HeaderHelper.isTrustedClusterRequest(threadContext)) {
threadContext.addResponseHeader(
ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER,
Base64Helper.serializeObject((Serializable) maskedFieldsMap)
);
if (log.isDebugEnabled()) {
log.debug("added response header for masked fields info: {}", maskedFieldsMap);
}
} else {

if (threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER) != null) {
if (!maskedFieldsMap.equals(
Base64Helper.deserializeObject(
threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER),
threadContext.getTransient(ConfigConstants.USE_JDK_SERIALIZATION)
)
)) {
throw new OpenSearchSecurityException(
ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER + " does not match (SG 901D)"
);
} else {
if (log.isDebugEnabled()) {
log.debug(ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER + " already set");
}
}
} else {
threadContext.putHeader(
ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER,
Base64Helper.serializeObject((Serializable) maskedFieldsMap)
);
if (log.isDebugEnabled()) {
log.debug("attach masked fields info: {}", maskedFieldsMap);
}
}
}
}

if (!flsRuleMap.isUnrestricted()) {
Map<String, Set<String>> flsFields = new HashMap<>();

for (Map.Entry<String, FieldPrivileges.FlsRule> entry : flsRuleMap.getIndexMap().entrySet()) {
flsFields.put(entry.getKey(), Sets.newHashSet(entry.getValue().getSource()));
}

if (request instanceof ClusterSearchShardsRequest && HeaderHelper.isTrustedClusterRequest(threadContext)) {
threadContext.addResponseHeader(
ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER,
Base64Helper.serializeObject((Serializable) flsFields)
);
if (log.isDebugEnabled()) {
log.debug("added response header for FLS info: {}", flsFields);
}
} else {
if (threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER) != null) {
if (!flsFields.equals(
Base64Helper.deserializeObject(
threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER),
threadContext.getTransient(ConfigConstants.USE_JDK_SERIALIZATION)
)
)) {
throw new OpenSearchSecurityException(
ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER
+ " does not match (SG 901D) "
+ flsFields
+ "---"
+ Base64Helper.deserializeObject(
threadContext.getHeader(ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER),
threadContext.getTransient(ConfigConstants.USE_JDK_SERIALIZATION)
)
);
} else {
if (log.isDebugEnabled()) {
log.debug(ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER + " already set");
}
}
} else {
threadContext.putHeader(
ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER,
Base64Helper.serializeObject((Serializable) flsFields)
);
if (log.isDebugEnabled()) {
log.debug("attach FLS info: {}", flsFields);
}
}
}

}
}

private static class BucketMerger implements Consumer<Bucket> {
private Comparator<MultiBucketsAggregation.Bucket> comparator;
private StringTerms.Bucket bucket = null;
Expand Down
Loading

0 comments on commit 5114d76

Please sign in to comment.