From 66062b64ab1e58a05e2a1ae533d0ba0c474166e2 Mon Sep 17 00:00:00 2001 From: Ronald Brill Date: Wed, 31 Jul 2024 18:10:15 +0200 Subject: [PATCH] WebRequest.getParameters() updated to deliver the same results as the servlet API (issue #836) --- src/changes/changes.xml | 7 + src/main/java/org/htmlunit/WebRequest.java | 21 +- .../java/org/htmlunit/WebRequest2Test.java | 298 ++++++++---------- .../java/org/htmlunit/WebRequestTest.java | 199 ------------ 4 files changed, 158 insertions(+), 367 deletions(-) diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 5d1e00c0ea..d0bc1ad3a1 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -8,6 +8,13 @@ + + WebRequest.getParameters() updated to deliver the same results as the servlet API + will report if this request reaches a servlet. This implies several changes: For url-encoded and + plain text encoded post requests, the query parameters are part of the result. + For multipart requests only the query parameters forming the result. Additionally only + the first occurence of a key is part of the result. + WorkerNavigator and WorkerLocation implemented. diff --git a/src/main/java/org/htmlunit/WebRequest.java b/src/main/java/org/htmlunit/WebRequest.java index da3828a367..46fd3364bc 100644 --- a/src/main/java/org/htmlunit/WebRequest.java +++ b/src/main/java/org/htmlunit/WebRequest.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -378,21 +379,28 @@ public List getParameters() { return normalize(allParameters); } - if (getEncodingType() == FormEncodingType.TEXT_PLAIN && HttpMethod.POST == getHttpMethod()) { + if (getEncodingType() == FormEncodingType.TEXT_PLAIN && HttpMethod.POST == getHttpMethod()) { if (getRequestBody() == null) { final List allParameters = new ArrayList<>(); allParameters.addAll(HttpUtils.parseUrlQuery(getUrl().getQuery(), getCharset())); - allParameters.addAll(getRequestParameters()); + // the servlet api ignores the parameters + // allParameters.addAll(getRequestParameters()); return normalize(allParameters); } return normalize(HttpUtils.parseUrlQuery(getUrl().getQuery(), getCharset())); } + if ((getEncodingType() == FormEncodingType.URL_ENCODED || getEncodingType() == FormEncodingType.TEXT_PLAIN) + && HttpMethod.PUT == getHttpMethod()) { + return normalize(HttpUtils.parseUrlQuery(getUrl().getQuery(), getCharset())); + } + if (FormEncodingType.MULTIPART == getEncodingType()) { final List allParameters = new ArrayList<>(); allParameters.addAll(HttpUtils.parseUrlQuery(getUrl().getQuery(), getCharset())); - allParameters.addAll(getRequestParameters()); + // the servlet api ignores the parameters + // allParameters.addAll(getRequestParameters()); return normalize(allParameters); } @@ -405,9 +413,14 @@ private static List normalize(final List pairs) { return pairs; } + final Set keys = new HashSet<>(); + final List resultingPairs = new ArrayList<>(); for (final NameValuePair pair : pairs) { - resultingPairs.add(pair.normalized()); + if (!keys.contains(pair.getName())) { + resultingPairs.add(pair.normalized()); + keys.add(pair.getName()); + } } return resultingPairs; diff --git a/src/test/java/org/htmlunit/WebRequest2Test.java b/src/test/java/org/htmlunit/WebRequest2Test.java index bb98d71c16..bea66fe35f 100644 --- a/src/test/java/org/htmlunit/WebRequest2Test.java +++ b/src/test/java/org/htmlunit/WebRequest2Test.java @@ -18,8 +18,9 @@ import java.io.Writer; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -29,15 +30,19 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.htmlunit.junit.BrowserRunner; +import org.apache.commons.lang3.StringUtils; +import org.htmlunit.junit.BrowserParameterizedRunner; +import org.htmlunit.junit.BrowserParameterizedRunner.Default; import org.htmlunit.util.NameValuePair; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; /** * Tests for {@link WebRequest#getParameters()}. - * This method is used by Spring MVC test to shortcut the + * This method is used by Spring test to shortcut the * request processing (without involving a server). Therefore * we have to make sure this works as expected in all cases. * @@ -46,9 +51,28 @@ * * @author Ronald Brill */ -@RunWith(BrowserRunner.class) +@RunWith(BrowserParameterizedRunner.class) public class WebRequest2Test extends WebServerTestCase { + public static class TestParameters { + private final String label_; + private final List pairs_; + + TestParameters(final String label, final List pairs) { + label_ = label; + pairs_ = pairs; + } + + @Override + public String toString() { + return label_; + } + + public List getPairs() { + return pairs_; + } + } + /** * Performs pre-test construction. * @throws Exception if an error occurs @@ -64,198 +88,144 @@ public void setup() throws Exception { } /** - * @throws Exception if the test fails + * Returns the parameterized data. + * @return the parameterized data + * @throws Exception if an error occurs */ - @Test - public void getParametersNone() throws Exception { - final WebRequest request = new WebRequest(URL_FIRST); - - final Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n", ((TextPage) page).getContent()); + @Parameters + public static Collection data() throws Exception { + final List data = new ArrayList<>(); + + final HttpMethod[] methods = {HttpMethod.OPTIONS, HttpMethod.GET, /* HttpMethod.HEAD,*/ HttpMethod.POST, + HttpMethod.PUT, HttpMethod.DELETE /* , HttpMethod.TRACE, HttpMethod.PATCH */}; + final String[] queries = {"", "?a=b", "?a=b&c=d", "?a=", "?a", "?", "?a=b&a=d"}; + final FormEncodingType[] encodings = + {FormEncodingType.URL_ENCODED, FormEncodingType.MULTIPART, FormEncodingType.TEXT_PLAIN}; + + List parameterPairs = new ArrayList<>(); + final TestParameters emptyParameters = new TestParameters("empty", parameterPairs); + + parameterPairs = new ArrayList<>(); + parameterPairs.add(new NameValuePair("p1", "v1")); + final TestParameters oneParameter = new TestParameters("oneParameter", parameterPairs); + + parameterPairs = new ArrayList<>(); + parameterPairs.add(new NameValuePair("a", "")); + final TestParameters emptyValueParameter = new TestParameters("emptyValue", parameterPairs); + + parameterPairs = new ArrayList<>(); + parameterPairs.add(new NameValuePair("a", "b")); + final TestParameters sameAsInQueryParameter = new TestParameters("sameAsInQuery", parameterPairs); + + final TestParameters[] parameters = + {null, emptyParameters, oneParameter, emptyValueParameter, sameAsInQueryParameter}; + + final String[] bodies = {"", "a=b", "a=b&c=d", "a=", "a", "a=b&a=d"}; + + for (final HttpMethod method : methods) { + for (final String query : queries) { + for (final FormEncodingType encoding : encodings) { + for (final TestParameters parameter : parameters) { + for (final String body : bodies) { + data.add(new Object[] {method, query, encoding, parameter, body}); + } + } + } + } + } - final List parameters = request.getParameters(); - assertEquals(0, parameters.size()); + return data; } /** - * @throws Exception if the test fails + * The HttpMethod. */ - @Test - public void getParametersOnePair() throws Exception { - final WebRequest request = new WebRequest(new URL(URL_FIRST, "?x=u")); - Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'x': 'u'\n", ((TextPage) page).getContent()); - - List parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("x", parameters.get(0).getName()); - assertEquals("u", parameters.get(0).getValue()); - - final List pairs = new ArrayList<>(); - pairs.add(new NameValuePair("hello", "world")); - request.setRequestParameters(pairs); - page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'hello': 'world'\n", ((TextPage) page).getContent()); - - parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("hello", parameters.get(0).getName()); - assertEquals("world", parameters.get(0).getValue()); - } + @Parameter + public HttpMethod httpMethod_; /** - * @throws Exception if the test fails + * The query. */ - @Test - public void getParametersOnePairKeyEquals() throws Exception { - final WebRequest request = new WebRequest(new URL(URL_FIRST, "?x=")); - Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'x': ''\n", ((TextPage) page).getContent()); - - List parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("x", parameters.get(0).getName()); - assertEquals("", parameters.get(0).getValue()); - - final List pairs = new ArrayList<>(); - pairs.add(new NameValuePair("hello", "")); - request.setRequestParameters(pairs); - page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'hello': ''\n", ((TextPage) page).getContent()); - - parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("hello", parameters.get(0).getName()); - assertEquals("", parameters.get(0).getValue()); - } + @Parameter(1) + public String query_; /** - * @throws Exception if the test fails + * The FormEncodingType. */ - @Test - public void getParametersOnePairKeyOnly() throws Exception { - final WebRequest request = new WebRequest(new URL(URL_FIRST, "?x")); - Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'x': ''\n", ((TextPage) page).getContent()); + @Parameter(2) + public FormEncodingType encoding_; - List parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("x", parameters.get(0).getName()); - assertEquals("", parameters.get(0).getValue()); - - final List pairs = new ArrayList<>(); - pairs.add(new NameValuePair("hello", null)); - request.setRequestParameters(pairs); - page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'hello': ''\n", ((TextPage) page).getContent()); + /** + * The FormEncodingType. + */ + @Parameter(3) + public TestParameters parameter_; - parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("hello", parameters.get(0).getName()); - assertEquals("", parameters.get(0).getValue()); - } + /** + * The body. + */ + @Parameter(4) + public String body_; /** - * @throws Exception if the test fails + * The default test. + * @throws Exception if an error occurs */ @Test - public void getParametersFromQueryAndUrlEncodedBodyPost() throws Exception { - final URL url = new URL(URL_FIRST, "?a=b"); + @Default + public void test() throws Exception { + final URL url = new URL(URL_FIRST, query_); final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.URL_ENCODED); - request.setRequestBody("c=d"); - - final Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - - assertEquals("Parameters: \n 'a': 'b'\n 'c': 'd'\n", ((TextPage) page).getContent()); + request.setHttpMethod(httpMethod_); + request.setEncodingType(encoding_); - final List parameters = request.getParameters(); - assertEquals(2, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); - assertEquals("c", parameters.get(1).getName()); - assertEquals("d", parameters.get(1).getValue()); - } + if (parameter_ != null) { + request.setRequestParameters(parameter_.getPairs()); + } - @Test - public void getParametersFromQueryAndUrlEncodedBodyPostWhenEncodingTypeIsMultipart() throws Exception { - final URL url = new URL(URL_FIRST, "?a=b"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.MULTIPART); - request.setRequestParameters(Collections.singletonList(new NameValuePair("c", "d"))); + if (body_ != null) { + try { + request.setRequestBody(body_); + } + catch (final RuntimeException e) { + // ignore + } + } final Page page = getWebClient().getPage(request); assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'a': 'b'\n", ((TextPage) page).getContent()); - final List parameters = request.getParameters(); - assertEquals(2, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); - assertEquals("c", parameters.get(1).getName()); - assertEquals("d", parameters.get(1).getValue()); - } + // calculate expectation from bounce servlet + String expectedContent = ((TextPage) page).getContent(); + assertTrue(expectedContent.startsWith("Parameters: \n")); + expectedContent = expectedContent.substring("Parameters: \n".length()).trim(); - @Test - public void getParametersUrlEncodedPostNoBody() throws Exception { - final URL url = new URL(URL_FIRST, "?a=b"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.URL_ENCODED); + final List expectedParameters = new ArrayList<>(); + if (!StringUtils.isAllBlank(expectedContent)) { + for (final String line : expectedContent.split("\n")) { + final String[] parts = line.split(":"); + assertEquals(2, parts.length); - final Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'a': 'b'\n", ((TextPage) page).getContent()); - - final List parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); - } - - @Test - public void getParametersTextEncodedPostNoBody() throws Exception { - final URL url = new URL(URL_FIRST, "?a=b"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.TEXT_PLAIN); + String name = parts[0].trim(); + name = StringUtils.strip(name, "'"); + String value = parts[1].trim(); + value = StringUtils.strip(value, "'"); - final Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'a': 'b'\n", ((TextPage) page).getContent()); + final NameValuePair pair = new NameValuePair(name, value); + expectedParameters.add(pair); + } + } final List parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); - } + assertEquals(expectedParameters.size(), parameters.size()); - @Test - public void getParametersTextEncodedPostBody() throws Exception { - final URL url = new URL(URL_FIRST, "?a=b"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.TEXT_PLAIN); - request.setRequestBody("c=d"); + final Iterator expectedIter = expectedParameters.iterator(); + for (final NameValuePair nameValuePair : parameters) { + final NameValuePair expectedNameValuePair = expectedIter.next(); - final Page page = getWebClient().getPage(request); - assertTrue(page instanceof TextPage); - assertEquals("Parameters: \n 'a': 'b'\n", ((TextPage) page).getContent()); - - final List parameters = request.getParameters(); - assertEquals(1, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); + assertEquals(expectedNameValuePair.getName(), nameValuePair.getName()); + assertEquals(expectedNameValuePair.getValue(), nameValuePair.getValue()); + } } /** diff --git a/src/test/java/org/htmlunit/WebRequestTest.java b/src/test/java/org/htmlunit/WebRequestTest.java index c60991ca01..9f1b7b020f 100644 --- a/src/test/java/org/htmlunit/WebRequestTest.java +++ b/src/test/java/org/htmlunit/WebRequestTest.java @@ -277,162 +277,6 @@ public void hiddenFileInWindows2() throws Exception { assertEquals(url.toExternalForm(), request.getUrl().toExternalForm()); } - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersNone() throws Exception { - final URL url = new URL("http://localhost/test"); - final WebRequest request = new WebRequest(url); - assertEquals(0, request.getParameters().size()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromUrlGet() throws Exception { - final URL url = new URL("http://localhost/test?x=u"); - final WebRequest request = new WebRequest(url); - - assertEquals(1, request.getParameters().size()); - assertEquals("x", request.getParameters().get(0).getName()); - assertEquals("u", request.getParameters().get(0).getValue()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromUrlKeyEqualsOnlyGet() throws Exception { - final URL url = new URL("http://localhost/test?x="); - final WebRequest request = new WebRequest(url); - - assertEquals(1, request.getParameters().size()); - assertEquals("x", request.getParameters().get(0).getName()); - assertEquals("", request.getParameters().get(0).getValue()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromUrlKeyOnlyGet() throws Exception { - final URL url = new URL("http://localhost/test?x"); - final WebRequest request = new WebRequest(url); - - assertEquals(1, request.getParameters().size()); - assertEquals("x", request.getParameters().get(0).getName()); - assertEquals("", request.getParameters().get(0).getValue()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromRequestParametersGet() throws Exception { - final URL url = new URL("http://localhost/test?x=u"); - final WebRequest request = new WebRequest(url); - - final List pairs = new ArrayList<>(); - request.setRequestParameters(pairs); - assertEquals(1, request.getParameters().size()); - assertEquals("x", request.getParameters().get(0).getName()); - assertEquals("u", request.getParameters().get(0).getValue()); - - pairs.add(new NameValuePair("hello", "world")); - assertEquals(1, request.getParameters().size()); - assertEquals("hello", request.getParameters().get(0).getName()); - assertEquals("world", request.getParameters().get(0).getValue()); - - // test for our internal conversation - String query = HttpUtils.toQueryFormFields(pairs, StandardCharsets.UTF_8); - assertEquals("http://localhost/test?hello=world", UrlUtils.toURI(url, query).toString()); - - pairs.add(new NameValuePair("empty", "")); - assertEquals(2, request.getParameters().size()); - assertEquals("empty", request.getParameters().get(1).getName()); - assertEquals("", request.getParameters().get(1).getValue()); - - // test for our internal conversation - query = HttpUtils.toQueryFormFields(pairs, StandardCharsets.UTF_8); - assertEquals("http://localhost/test?hello=world&empty=", UrlUtils.toURI(url, query).toString()); - - pairs.add(new NameValuePair("null", null)); - assertEquals(3, request.getParameters().size()); - assertEquals("null", request.getParameters().get(2).getName()); - assertEquals("", request.getParameters().get(2).getValue()); - - // test for our internal conversation - query = HttpUtils.toQueryFormFields(pairs, StandardCharsets.UTF_8); - assertEquals("http://localhost/test?hello=world&empty=&null", UrlUtils.toURI(url, query).toString()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromUrlPost() throws Exception { - final URL url = new URL("http://localhost/test?x=u"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - - assertEquals(0, request.getParameters().size()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromUrlEncodedBodyPost() throws Exception { - final URL url = new URL("http://localhost/test"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.URL_ENCODED); - request.setRequestBody("x=u"); - - assertEquals(1, request.getParameters().size()); - assertEquals("x", request.getParameters().get(0).getName()); - assertEquals("u", request.getParameters().get(0).getValue()); - } - - /** - * @throws Exception if the test fails - */ - @Test - public void getParametersFromQueryAndUrlEncodedBodyPost() throws Exception { - final URL url = new URL("http://localhost/test?a=b"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.URL_ENCODED); - request.setRequestBody("c=d"); - - final List parameters = request.getParameters(); - - assertEquals(2, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); - assertEquals("c", parameters.get(1).getName()); - assertEquals("d", parameters.get(1).getValue()); - } - - @Test - public void getParametersFromQueryAndUrlEncodedBodyPostWhenEncodingTypeIsMultipart() throws Exception { - final URL url = new URL("http://localhost/test?a=b"); - final WebRequest request = new WebRequest(url); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.MULTIPART); - request.setRequestParameters(Collections.singletonList(new NameValuePair("c", "d"))); - - final List parameters = request.getParameters(); - - assertEquals(2, parameters.size()); - assertEquals("a", parameters.get(0).getName()); - assertEquals("b", parameters.get(0).getValue()); - assertEquals("c", parameters.get(1).getName()); - assertEquals("d", parameters.get(1).getValue()); - } - /** * @throws Exception if the test fails */ @@ -492,47 +336,4 @@ public void getRequestParametersFromUrlEncodedBodyPost() throws Exception { assertEquals(0, request.getRequestParameters().size()); } - - @Test - public void getParametersShouldNotModifyAlreadyNormalizedRequestParams() throws Exception { - final WebRequest request = new WebRequest(new URL("http://localhost/test")); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.MULTIPART); - - final List requestParams = new ArrayList<>(); - requestParams.add(new NameValuePair("test", "x")); - requestParams.add(new KeyDataPair("file", - new File("test"), - "test", - "application/octet-stream", - StandardCharsets.UTF_8)); - request.setRequestParameters(requestParams); - - //check that the result of getParams is equal to the requestParams after "normalization" - assertEquals(requestParams, request.getParameters()); - } - - @Test - public void getParametersShouldNormalizeMultiPartRequestParams() throws Exception { - final WebRequest request = new WebRequest(new URL("http://localhost/test")); - request.setHttpMethod(HttpMethod.POST); - request.setEncodingType(FormEncodingType.MULTIPART); - - final List requestParams = new ArrayList<>(); - requestParams.add(new NameValuePair("test", null)); - requestParams.add(new KeyDataPair("file", null, null, null, StandardCharsets.UTF_8)); - request.setRequestParameters(requestParams); - - final List expectedResults = new ArrayList<>(); - expectedResults.add(new NameValuePair("test", "")); - // the constructor of the KeyDataPair already creates normalized object - // where the value is set to empty string if the passed file is null. - expectedResults.add(new KeyDataPair("file", null, null, null, StandardCharsets.UTF_8)); - - final List normalizedParams = request.getParameters(); - assertEquals(expectedResults, normalizedParams); - - // check that the value of the KeyDataPair is really normalized to empty string - assertEquals("", normalizedParams.get(1).getValue()); - } }