Skip to content

RANGER-5406 Support export policies in a segmented manner #748

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yunyezhang-work wants to merge 1 commit into
base: master
Choose a base branch
from
+84 −1
Open

RANGER-5406 Support export policies in a segmented manner #748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ public class SearchFilter {
public static final String CREATE_TIME = "createTime"; // sort
public static final String UPDATE_TIME = "updateTime"; // sort
public static final String START_INDEX = "startIndex";
public static final String BEGIN_INDEX = "beginIndex";
public static final String OFFSET = "offset";
public static final String PAGE_SIZE = "pageSize";
public static final String SORT_BY = "sortBy";
public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search
Expand Down Expand Up @@ -152,6 +154,8 @@ public class SearchFilter {
private Map<String, Object[]> multiValueParams;
private int startIndex;
private int maxRows = Integer.MAX_VALUE;
private int beginIndex = 0;
private int offset = -1;
private boolean getCount = true;
private String sortBy;
private String sortType;
Expand All @@ -167,6 +171,8 @@ public SearchFilter(SearchFilter other) {
setMultiValueParams(other.multiValueParams != null ? new HashMap<>(other.multiValueParams) : null);
setStartIndex(other.startIndex);
setMaxRows(other.maxRows);
setBeginIndex(other.beginIndex);
setOffset(other.offset);
setGetCount(other.getCount);
setSortBy(other.sortBy);
setSortType(other.sortType);
Expand Down Expand Up @@ -272,6 +278,23 @@ public void setStartIndex(int startIndex) {
this.startIndex = startIndex;
}

public int getBeginIndex() {
return beginIndex;
}

public void setBeginIndex(int beginIndex) {
this.beginIndex = beginIndex;
}

public int getOffset() {
return offset;
}

public void setOffset(int offset) {
this.offset = offset;
}


public int getMaxRows() {
return maxRows;
}
Expand Down
18 changes: 17 additions & 1 deletion security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,14 +239,30 @@ public SearchFilter extractCommonCriteriasForFilter(HttpServletRequest request,
"Invalid value for parameter startIndex", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.START_INDEX);

startIndex = startIndex < 0 ? 0 : startIndex;
logger.info("==> setStartIndex={}" , startIndex);

ret.setStartIndex(startIndex);

int pageSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.PAGE_SIZE), configUtil.getDefaultMaxRows(),
"Invalid value for parameter pageSize", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.PAGE_SIZE);

logger.info("==> setMaxRows={}" , pageSize);
logger.info("==> DefaultMaxRows={}" , configUtil.getDefaultMaxRows());
ret.setMaxRows(validatePageSize(pageSize));

int beginIndex = restErrorUtil.parseInt(request.getParameter(SearchFilter.BEGIN_INDEX), 0,
"Invalid value for parameter beginIndex", MessageEnums.INVALID_INPUT_DATA, null,
SearchFilter.BEGIN_INDEX);
beginIndex = beginIndex < 0 ? startIndex : beginIndex;
logger.info("==> setBeginIndex={}" , beginIndex);
ret.setBeginIndex(beginIndex);

int offsetSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.OFFSET), 0,
"Invalid value for parameter offset", MessageEnums.INVALID_INPUT_DATA, null,
SearchFilter.OFFSET);
logger.info("==> setOffsetIndex={}" , offsetSize);
offsetSize = offsetSize < 0 ? pageSize : offsetSize;
ret.setOffset(offsetSize);

if (request.getParameter(SearchFilter.POLICY_TYPE) != null) {
int policyType = restErrorUtil.parseInt(request.getParameter(SearchFilter.POLICY_TYPE), 0,
"Invalid value for parameter policyType", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.POLICY_TYPE);
Expand Down
44 changes: 44 additions & 0 deletions security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -3729,6 +3730,8 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
List<String> serviceTypeList = null;
List<String> serviceNameInServiceTypeList = new ArrayList<>();
boolean isServiceExists;
int begin = 0;
int offset = -1;

if (request.getParameter(PARAM_SERVICE_NAME) != null) {
serviceNames = request.getParameter(PARAM_SERVICE_NAME);
Expand All @@ -3750,6 +3753,9 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
List<RangerPolicy> policyListByServiceName = new ArrayList<>();

if (filter != null) {
begin = filter.getBeginIndex();
offset = filter.getOffset();
LOG.info("==> beginIndex: {}, offset: {}", begin, offset);
filter.setStartIndex(0);
filter.setMaxRows(Integer.MAX_VALUE);

Expand Down Expand Up @@ -3830,7 +3836,12 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
policyLists.addAll(orderedPolicies.values());
}
}
LOG.info("<==policyLists size:{}", policyLists.size());

if(begin>=0 && offset >0) {
policyLists = getRangerPoliciesInRange(policyLists, filter);
LOG.info("<==policyLists size after cut:{}" , policyLists.size());
}
return policyLists;
}

Expand Down Expand Up @@ -4298,6 +4309,39 @@ private RangerPolicyList toRangerPolicyList(List<RangerPolicy> policyList, Searc
return ret;
}

private List<RangerPolicy> getRangerPoliciesInRange(List<RangerPolicy> policyList, SearchFilter filter) {
if (CollectionUtils.isEmpty(policyList)) {
return Collections.emptyList();
}

int totalCount = policyList.size();
int startIndex = filter.getBeginIndex();
int offset = filter.getOffset();
int toIndex = Math.min(startIndex + offset, totalCount);
LOG.info("==>totalCount: {}, startIndex:{}, offsetSize: {}, toIndex: {}" , totalCount, startIndex, offset, toIndex);
String sortType = filter.getSortType();
String sortBy = filter.getSortBy();

if (StringUtils.isNotEmpty(sortBy) && StringUtils.isNotEmpty(sortType)) {
// By default policyList is sorted by policyId in asc order, So handling only desc case.
if (SearchFilter.POLICY_ID.equalsIgnoreCase(sortBy)) {
if (SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
policyList.sort(this.getPolicyComparator(sortBy, sortType));
}
} else if (SearchFilter.POLICY_NAME.equalsIgnoreCase(sortBy)) {
if (SORT_ORDER.ASC.name().equalsIgnoreCase(sortType) || SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
policyList.sort(this.getPolicyComparator(sortBy, sortType));
} else {
LOG.info("Invalid or Unsupported sortType : {}", sortType);
}
} else {
LOG.info("Invalid or Unsupported sortBy property : {}" , sortBy);
}
}

return new ArrayList<>(policyList.subList(startIndex, toIndex));
}

private Comparator<RangerPolicy> getPolicyComparator(String sortBy, String sortType) {
return (RangerPolicy me, RangerPolicy other) -> {
int ret = 0;
Expand Down