Gist for "Use cases for the readUrl function in DataWeave" blog-post

categorized_functions_exposed_by_library.dwl

---
{
  functionsOnObjects: {
    getUniqueKeys: getUniqueKeys,
    getKeys: getKeys
  },
  functionsOnStrings: {
    substring: substringFunc
  },
  functionsOnArrays: {
    tail: tail,
    subArray: subArray
  }
}

countries.csv

countries_dw_transformation.dwl

%dw 1.0
%output application/json
%var countries = readUrl("classpath://countries.csv", "application/csv")
%var countryDetails = (countries filter $.name == payload.country)[0]
---
country: {
    name : payload.country,
    code : countryDetails.alpha-3,
    region: countryDetails.sub-region
}

countries_input_payload.json

{"country" : "Andorra"}

countries_output_payload.json

{"country":
  {
    "name": "Andorra",
    "code": "AND",
    "region": "Southern Europe"
  }
}

dw_transformation_readurl_library.dwl

%dw 1.0
%output application/json
%var myLibrary = readUrl("classpath://library.dwl","application/dw")
---
{
  sub: myLibrary.substring(payload.myString,3,6),
  keys: myLibrary.getUniqueKeys(payload.myObject)
}

etl_flow_using_choice_router.xml

<flow name="etl-flow">
  <file:inbound-endpoint  doc:name="Extract Entity File" path="/path/to/files_to_extract" responseTimeout="10000"/>
  <choice doc:name="Choice">
      <when expression="#[payload.type == &quot;Person&quot;]">
          <dw:transform-message doc:name="Transform Person File">
              <dw:set-payload resource="classpath:Person.dwl"/>
          </dw:transform-message>
      </when>
      <when expression="#[payload.type == &quot;Location&quot;]">
          <dw:transform-message doc:name="Transform Location File">
              <dw:set-payload resource="classpath:Location.dwl"/>
          </dw:transform-message>
      </when>
      <when expression="#[payload.type == &quot;Product&quot;]">
          <dw:transform-message doc:name="Transform Product File">
              <dw:set-payload resource="classpath:Product.dwl"/>
          </dw:transform-message>
      </when>
      <otherwise>
          <logger level="INFO" doc:name="Logger"/>
      </otherwise>
  </choice>
  <flow-ref doc:name="Load Entity Subflow" name="load-entity-subflow"/>
</flow>

etl_flow_using_readurl.xml

<flow name="etl-flow">
  <file:inbound-endpoint doc:name="Extract Entity File" path="/path/to/files_to_extract" responseTimeout="10000"/>
  <dw:transform-message doc:name="Transform Entity File">
      <dw:set-payload><![CDATA[
      %dw 1.0
      %output application/java
      %var transform = readUrl("classpath://" ++ payload.type ++ ".dwl", "application/dw")
      ---	   
      transform(payload)
  ]]></dw:set-payload>
  </dw:transform-message>
  <flow-ref doc:name="Load Entity Subflow" name="load-entity-subflow"/>
</flow>

input_payload_readurl_library.json

{
  "myString": "Hello World!",
  "myObject": {"a": 1, "b": 2, "a": 3, "c": 4}
}

library.dwl

%dw 1.0
%output application/dw
//I suggest using application/dw as the output type, so that any possible output will be supported.
//Anyhow, the output MIME type used by the callers of this library will take precedence.

%function getUniqueKeys(obj)
getKeys(obj) distinctBy $

%function getKeys(obj)
obj pluck $$

%function tail(arr)
[] when (sizeOf arr) <= 1 otherwise arr[1..-1] 

%function substringFunc(str,i,j) 
str splitBy "" filter ($$>=i and $$<=j) joinBy "" 

%function subArray(arr,i,j) 
arr filter ($$>=posi and $$<=posj)

---

//Mapping the function names to the functions themselves.
//The exposed function name can be different from the actual function name,
//which is the case for the function substringFunc, whose exposed name is substring.
//Also, note that not all functions may be exposed, such as the getKeys function in this case.
{
  getUniqueKeys: getUniqueKeys,
  tail: tail,
  substring: substringFunc,
  subArray: subArray
}

output_payload_readurl_library.json

{
  "sub": "lo W",
  "keys": ["a","b","c"]
}

read_dw_transformation.dwl

%dw 1.0
%input payload application/java
%output application/dw
---
read(payload, "application/csv")

read_dw_transformation2.dwl

%dw 1.0
%output application/json
%var data = payload.data
---
payload.transformations mapObject
{($$) : read(data ++ " " ++ $)}

read_input_payload

"name,surname,id\nJohn,Smith,574323\nMary,Cooper,729852"

read_input_payload2.json

{
  "data": "[2,6,3,6,1]",
  "transformations":
  {
    "plusOne": "map \$+1",
    "removeDuplicatesAndSort": "distinctBy \$ orderBy \$",
    "filterOutOddIndexes": "filter (\$\$ mod 2) == 0"
  }
}

read_output_payload.json

[
  {
    "name": "John",
    "surname": "Smith",
    "id": "574323"
  },
  {
    "name": "Mary",
    "surname": "Cooper",
    "id": "729852"
  }
]

read_output_payload2.json

{
  "plusOne": [3,7,4,7,2],
  "removeDuplicatesAndSort": [1,2,3,6],
  "filterOutOddIndexes": [2,3,1]
}