Azure WASB クライアントのテスト

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.azure.wasb.account.name</name>
    <value>{ACCOUNTNAME}.blob.core.windows.net</value>
  </property>
  <property>
    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
    <value>{ACCOUNT ACCESS KEY}</value>
  </property>
</configuration>

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.contract.test.fs.wasb</name>
    <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
    <description>The name of the azure file system for testing.</description>
  </property>
  <property>
    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
    <value>{ACCOUNT ACCESS KEY}</value>
  </property>
</configuration>

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.azure.wasb.account.name</name>
    <value>{ACCOUNTNAME}.blob.core.windows.net</value>
  </property>
  <property>
    <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
    <value>{ACCOUNT ACCESS KEY}</value>
  </property>
  <property>
    <name>fs.contract.test.fs.wasb</name>
    <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
  </property>
</configuration>

mvn -T 1C clean verify

mvn -T 1C -Dparallel-tests=both clean verify
mvn -T 1C -Dparallel-tests=wasb clean verify
mvn -T 1C -Dparallel-tests=abfs clean verify

mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean verify

mvn -T 1C clean test

mvn -T 1C -Dparallel-tests clean test

mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean test

mvn -T 1C clean test -Dtest=TestRollingWindowAverage

mvn -T 1C clean verify -Dscale -Dit.test=ITestFileSystemOperationExceptionMessage -Dtest=none

mvn -T 1C clean verify -Dtest=none -Dit.test=ITest*

# for the unit tests
mvn -T 1C surefire-report:report-only

# for the integration tests
mvn -T 1C surefire-report:failsafe-report-only

# all reports for this module
mvn -T 1C site:site

<property>
  <name>fs.azure.test.emulator</name>
  <value>true</value>
</property>

com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.

log4j.logger.org.apache.hadoop.fs.azure=DEBUG

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
  <property>
    <name>fs.azure.abfs.account.name</name>
    <value>{ACCOUNT_NAME}.dfs.core.windows.net</value>
  </property>

  <property>
    <name>fs.azure.account.key.{ACCOUNT_NAME}.dfs.core.windows.net</name>
    <value>{ACCOUNT_ACCESS_KEY}</value>
  </property>

  <property>
    <name>fs.azure.wasb.account.name</name>
    <value>{ACCOUNT_NAME}.blob.core.windows.net</value>
  </property>

  <property>
    <name>fs.azure.account.key.{ACCOUNT_NAME}.blob.core.windows.net</name>
    <value>{ACCOUNT_ACCESS_KEY}</value>
  </property>

  <property>
    <name>fs.contract.test.fs.abfs</name>
    <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
    <description>A file system URI to be used by the contract tests.</description>
  </property>

  <property>
    <name>fs.contract.test.fs.wasb</name>
    <value>wasb://{CONTAINER_NAME}@{ACCOUNT_NAME}.blob.core.windows.net</value>
    <description>A file system URI to be used by the contract tests.</description>
  </property>
</configuration>

<!--=========================== AUTHENTICATION  OPTIONS ===================-->
<!--ATTENTION:
      TO RUN ABFS & WASB COMPATIBILITY TESTS, YOU MUST SET AUTH TYPE AS SharedKey.
      OAUTH IS INTRODUCED TO ABFS ONLY.-->
<property>
  <name>fs.azure.account.auth.type.{YOUR_ABFS_ACCOUNT_NAME}</name>
  <value>{AUTH TYPE}</value>
  <description>The authorization type can be SharedKey, OAuth, Custom or SAS. The
  default is SharedKey.</description>
</property>

<!--=============================   FOR OAUTH   ===========================-->
<!--IF AUTH TYPE IS SET AS OAUTH, FOLLOW THE STEPS BELOW-->
<!--NOTICE: AAD client and tenant related properties can be obtained through Azure Portal-->

  <!--1. UNCOMMENT BELOW AND CHOOSE YOUR OAUTH PROVIDER TYPE -->

  <!--
  <property>
    <name>fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}</name>
    <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
    <description>The full name of token provider class name.</description>
  </property>
 -->

  <!--2. UNCOMMENT BELOW AND SET CREDENTIALS ACCORDING TO THE PROVIDER TYPE-->

  <!--2.1. If "ClientCredsTokenProvider" is set as key provider, uncomment below and
           set auth endpoint, client id and secret below-->
  <!--
   <property>
    <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
    <description>Token end point, this can be found through Azure portal</description>
  </property>

   <property>
     <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
     <value>{client id}</value>
     <description>AAD client id.</description>
   </property>

   <property>
     <name>fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}</name>
     <value>{client secret}</value>
   </property>
 -->

  <!--2.2. If "UserPasswordTokenProvider" is set as key provider, uncomment below and
           set auth endpoint, use name and password-->
  <!--
   <property>
    <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
    <description>Token end point, this can be found through Azure portal</description>
  </property>

   <property>
     <name>fs.azure.account.oauth2.user.name.{ABFS_ACCOUNT_NAME}</name>
     <value>{user name}</value>
   </property>

   <property>
     <name>fs.azure.account.oauth2.user.password.{ABFS_ACCOUNT_NAME}</name>
     <value>{user password}</value>
   </property>
 -->

  <!--2.3. If "MsiTokenProvider" is set as key provider, uncomment below and
           set tenantGuid and client id.-->
  <!--
   <property>
     <name>fs.azure.account.oauth2.msi.tenant.{ABFS_ACCOUNT_NAME}</name>
     <value>{tenantGuid}</value>
     <description>msi tenantGuid.</description>
   </property>

   <property>
     <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
     <value>{client id}</value>
     <description>AAD client id.</description>
   </property>
  -->

  <!--2.4. If "RefreshTokenBasedTokenProvider" is set as key provider, uncomment below and
           set refresh token and client id.-->
  <!--
   <property>
     <name>fs.azure.account.oauth2.refresh.token.{ABFS_ACCOUNT_NAME}</name>
     <value>{refresh token}</value>
     <description>refresh token.</description>
   </property>

   <property>
     <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
     <value>{client id}</value>
     <description>AAD client id.</description>
   </property>
  -->

  <!--
    <property>
        <name>fs.azure.identity.transformer.enable.short.name</name>
        <value>true/false</value>
        <description>
          User principal names (UPNs) have the format “{alias}@{domain}”.
          If true, only {alias} is included when a UPN would otherwise appear in the output
          of APIs like getFileStatus, getOwner, getAclStatus, etc, default is false.
        </description>
    </property>

    <property>
        <name>fs.azure.identity.transformer.domain.name</name>
        <value>domain name of the user's upn</value>
        <description>
          If the domain name is specified and “fs.azure.identity.transformer.enable.short.name”
          is true, then the {alias} part of a UPN can be specified as input to APIs like setOwner,
          setAcl, modifyAclEntries, or removeAclEntries, and it will be transformed to a UPN by appending @ and the domain specified by
          this configuration property.
        </description>
    </property>

    <property>
        <name>fs.azure.identity.transformer.service.principal.id</name>
        <value>service principal object id</value>
        <description>
          An Azure Active Directory object ID (oid) used as the replacement for names contained
          in the list specified by “fs.azure.identity.transformer.service.principal.substitution.list”.
          Notice that instead of setting oid, you can also set $superuser here.
        </description>
    </property>

    <property>
        <name>fs.azure.identity.transformer.skip.superuser.replacement</name>
        <value>true/false</value>
        <description>
          If false, “$superuser” is replaced with the current user when it appears as the owner
          or owning group of a file or directory. The default is false.
        </description>
    </property>

    <property>
        <name>fs.azure.identity.transformer.service.principal.substitution.list</name>
        <value>mapred,hdfs,yarn,hive,tez</value>
        <description>
           A comma separated list of names to be replaced with the service principal ID specified by
           “fs.azure.identity.transformer.service.principal.id”.  This substitution occurs
           when setOwner, setAcl, modifyAclEntries, or removeAclEntries are invoked with identities
           contained in the substitution list. Notice that when in non-secure cluster, asterisk symbol *
           can be used to match all user/group.
        </description>
    </property>
   -->

<!--=========================== AUTHENTICATION  OPTIONS ===================-->
<!--=============================   FOR SAS   ===========================-->
<!-- To run ABFS Delegation SAS tests, you must register an app, create the
     necessary role assignments, and set the configuration discussed below:

    1) Register an app:
      a) Login to https://portal.azure.com, select your AAD directory and search for app registrations.
      b) Click "New registration".
      c) Provide a display name, such as "abfs-app".
      d) Set the account type to "Accounts in this organizational directory only ({YOUR_Tenant} only - Single tenant)".
      e) For Redirect URI select Web and enter "http://localhost".
      f) Click Register.

    2)  Create necessary role assignments:
      a) Login to https://portal.azure.com and find the Storage account with hierarchical namespace enabled
         that you plan to run the tests against.
      b) Select "Access Control (IAM)".
      c) Select Role Assignments
      d) Click Add and select "Add role assignments"
      e) For Role and enter "Storage Blob Data Owner".
      f) Under Select enter the name of the app you registered in step 1 and select it.
      g) Click Save.
      h) Repeat above steps to create a second role assignment for the app but this time for
         the "Storage Blob Delegator" role.

    3) Generate a new client secret for the application:
      a) Login to https://portal.azure.com and find the app registered in step 1.
      b) Select "Certificates and secrets".
      c) Click "New client secret".
      d) Enter a description (eg. Secret1)
      e) Set expiration period.  Expires in 1 year is good.
      f) Click Add
      g) Copy the secret and expiration to a safe location.

    4) Set the following configuration values:
-->

  <property>
    <name>fs.azure.sas.token.provider.type</name>
    <value>org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider</value>
    <description>The fully qualified class name of the SAS token provider implementation.</description>
  </property>

  <property>
    <name>fs.azure.test.app.service.principal.tenant.id</name>
    <value>{TID}</value>
    <description>Tenant ID for the application's service principal.</description>
  </property>

  <property>
    <name>fs.azure.test.app.service.principal.object.id</name>
    <value>{OID}</value>
    <description>Object ID for the application's service principal.</description>
  </property>

  <property>
    <name>fs.azure.test.app.id</name>
    <value>{app id}</value>
    <description>The application's ID, also known as the client id.</description>
  </property>

   <property>
     <name>fs.azure.test.app.secret</name>
     <value>{client secret}</value>
     <description>The application's secret, also known as the client secret.</description>
   </property>

<!--===========================   FOR CheckAccess =========================-->
<!-- To run ABFS CheckAccess SAS tests, you must register an app, with no role
 assignments, and set the configuration discussed below:

    1) Register a new app with no RBAC
    2) As part of the test configs you need to provide the guid for the above
created app. Please follow the below steps to fetch the guid.
      a) Get an access token with the above created app. Please refer the
 following documentation for the same. https://docs.microsoft
.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
      b) Decode the token fetched with the above step. You may use https
://jwt.ms/ to decode the token
      d) The oid field in the decoded string is the guid.
    3) Set the following configurations:
-->

  <property>
    <name>fs.azure.enable.check.access</name>
    <value>true</value>
    <description>By default the check access will be on. Checkaccess can
    be turned off by changing this flag to false.</description>
  </property>
  <property>
    <name>fs.azure.account.test.oauth2.client.id</name>
    <value>{client id}</value>
    <description>The client id(app id) for the app created on step 1
    </description>
  </property>
  <property>
    <name>fs.azure.account.test.oauth2.client.secret</name>
    <value>{client secret}</value>
    <description>
The client secret(application's secret) for the app created on step 1
    </description>
  </property>
  <property>
    <name>fs.azure.check.access.testuser.guid</name>
    <value>{guid}</value>
    <description>The guid fetched on step 2</description>
  </property>
  <property>
    <name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
.windows.net</name>
    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
    <description>
Token end point. This can be found through Azure portal. As part of CheckAccess
test cases. The access will be tested for an FS instance created with the
above mentioned client credentials. So this configuration is necessary to
create the test FS instance.
    </description>
  </property>

<property>
  <name>fs.azure.abfs.endpoint</name>
  <value>{IP}:{PORT}</value>
</property>