天天看點

DataWorks OpenAPI 示例(中繼資料子產品)DataWorks OpenAPI 中繼資料demo

以下demo内容已經較老,大家使用API建議移步官方API文檔: https://help.aliyun.com/document_detail/173915.html 有詳細的demo代碼和線上調試功能

DataWorks OpenAPI 中繼資料demo

Client初始化

String region = "cn-shanghai";
    IClientProfile profile = DefaultProfile.getProfile(region, "xxx", "xxx");
    DefaultProfile.addEndpoint(region,"dataworks-public", String.format("dataworks.%s.aliyuncs.com", region));
    IAcsClient client = new DefaultAcsClient(profile);           

GetMetaDBInfo

  • 擷取odps引擎詳情
GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
    //odps app guid,格式 odps.{projectName}
    request.setAppGuid("odps.project1");
    //類型為odps
    request.setDataSourceType("odps");
    GetMetaDBInfoResponse res = client.getAcsResponse(request);
    GetMetaDBInfoResponse.Data data = res.getData();           
  • 擷取emr庫詳情
GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
    request.setDataSourceType("emr");
    //emr 叢集id
    request.setClusterId("C-C206DE8CACxxxxx");
    //emr db名稱
    request.setDatabaseName("default");

    GetMetaDBInfoResponse res = client.getAcsResponse(request);
    GetMetaDBInfoResponse.Data data = res.getData();           

ListMetaDB

  • 擷取emr 資料庫清單
ListMetaDBRequest request = new ListMetaDBRequest();
    //資料類型
    request.setDataSourceType("emr");
    //dataworks 項目id
    request.setProjectId(110L);
    request.setPageSize(1);
    request.setPageNum(10);

    ListMetaDBResponse res = client.getAcsResponse(request);
    //總條數
    res.getDatabaseInfo().getTotalCount();
    //資料
    res.getDatabaseInfo().getDbList();           

GetMetaDBTableList

  • 擷取odps引擎關聯表
GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
    //app guid
    request.setAppGuid("odps.project1");
    request.setDataSourceType("odps");
    //第1頁
    request.setPageNumber(1);
    //每頁大小
    request.setPageSize(10);
    GetMetaDBTableListResponse res =  client.getAcsResponse(request);
    //資料總大小
    Long totalCount = res.getData().getTotalCount();
    //目前頁資料清單
    List<GetMetaDBTableListResponse.Data.TableEntityListItem>  tableEntityListItems = res.getData().getTableEntityList();           
  • 擷取emr 資料庫下的表
GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CAC0xxxxx");
    //emr 資料庫名稱
    request.setDatabaseName("default");

    GetMetaDBTableListResponse res = client.getAcsResponse(request);
    //資料總大小
    Long totalCount = res.getData().getTotalCount();
    //目前頁資料清單
    List<GetMetaDBTableListResponse.Data.TableEntityListItem>  tableEntityListItems = res.getData().getTableEntityList();           

CheckMetaTable

  • 檢查odps表是否存在
CheckMetaTableRequest request = new CheckMetaTableRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //資源類型
    request.setDataSourceType("odps");
    CheckMetaTableResponse res = client.getAcsResponse(request);
    //表是否存在
    Boolean exists = res.getData();           
  • 檢查emr表是否存在
CheckMetaTableRequest request = new CheckMetaTableRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");

    CheckMetaTableResponse res = client.getAcsResponse(request);           

CheckMetaPartition

  • 檢查odps分區是否存在
CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //資源類型
    request.setDataSourceType("odps");
    //分區
    request.setPartition("dt_time=20200419");

    CheckMetaPartitionResponse res = client.getAcsResponse(request);
    //分區是否存在
    Boolean exists = res.getData();           
  • 檢查emr分區是否存在
CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");

    request.setPartition("ds=202001");

    CheckMetaPartitionResponse res = client.getAcsResponse(request);
    //分區是否存在
    Boolean exists = res.getData();           

SearchMetaTables

  • 搜尋odps表
SearchMetaTablesRequest request = new SearchMetaTablesRequest();
    //odps app guid,格式 odps.{projectName}
    request.setAppGuid("odps.p1");
    request.setDataSourceType("odps");
    //表類型,0:table, 1:view  預設全部
    request.setEntityType(0);
    //關鍵字
    request.setKeyword("test");
    //分頁
    request.setPageNumber(1);
    request.setPageSize(10);

    SearchMetaTablesResponse res = client.getAcsResponse(request);
    //資料總條數
    Long totalCount = res.getData().getTotalCount();
    //目前頁表清單
    res.getData().getDataEntityList();           
  • 搜尋emr表
SearchMetaTablesRequest request = new SearchMetaTablesRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    request.setKeyword("test");
    request.setPageNumber(1);
    request.setPageSize(10);

    SearchMetaTablesResponse res = client.getAcsResponse(request);
    //資料總條數
    Long totalCount = res.getData().getTotalCount();
    //目前頁表清單
    res.getData().getDataEntityList();           

GetMetaTableBasicInfo

  • 擷取odps表基本資訊
GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //資源類型
    request.setDataSourceType("odps");

    GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
    //表資料
    GetMetaTableBasicInfoResponse.Data info = res.getData();           
  • 擷取emr表基礎資訊
GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");

    GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
    //表資料
    GetMetaTableBasicInfoResponse.Data info = res.getData();           

GetMetaTableColumn

  • 擷取odps column清單
GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //資源類型
    request.setDataSourceType("odps");
    //分頁
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaTableColumnResponse res = client.getAcsResponse(request);
    //資料總條數
    Long totalCount = res.getData().getTotalCount();
    //目前頁資料清單
    List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();
           
  • 擷取emr表column資訊
GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaTableColumnResponse res = client.getAcsResponse(request);
    //資料總條數
    Long totalCount = res.getData().getTotalCount();
    //目前頁資料清單
    List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();           

GetMetaTablePartition

  • 擷取odps表分區
GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //資源類型
    request.setDataSourceType("odps");
    //分頁
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTablePartitionResponse res = client.getAcsResponse(request);
    //資料總條數
    res.getData().getTotalCount();
    //分區清單
    res.getData().getDataEntityList();           
  • 擷取emr表分區
GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");
    request.setPageNumber(1);
    request.setPageSize(10);

    GetMetaTablePartitionResponse res = client.getAcsResponse(request);
    //資料總條數
    res.getData().getTotalCount();
    //分區清單
    res.getData().getDataEntityList();           

GetMetaTableOutput

  • 擷取odps表輸出
GetMetaTableOutputRequest request = new GetMetaTableOutputRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //output開始時間
    request.setStartDate("2020-05-27");
    //output結束時間
    request.setEndDate("2020-06-12");
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableOutputResponse res = client.getAcsResponse(request);
    //資料總條數
    res.getData().getTotalCount();
    //輸出清單
    res.getData().getDataEntityList();           

GetMetaTableChangeLog

  • 擷取odps表變更資訊
GetMetaTableChangeLogRequest request = new GetMetaTableChangeLogRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //開始時間,格式yyyy-MM-dd HH:mm:ss
    request.setStartDate("2020-01-01 01:01:00");
    //格式yyyy-MM-dd HH:mm:ss
    request.setEndDate("2020-01-02 01:01:00");
    /**
     * 類型:
     * CREATE_TABLE
     * ALTER_TABLE
     * DROP_TABLE
     * ADD_PARTITION
     * DROP_PARTITION
     * ALTER_OWNER_ID
     * ALTER_LIFE_CYCLE_TIME
     */
    request.setChangeType("CREATE_TABLE");
    //對象 類型 TABLE,PARTITION
    request.setObjectType("TABLE");
    //分頁
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableChangeLogResponse res = client.getAcsResponse(request);
    //資料總條數
    res.getData().getTotalCount();
    //資料
    res.getData().getDataEntityList();           

GetMetaTableIntroWiki

  • 擷取wiki
GetMetaTableIntroWikiRequest request = new GetMetaTableIntroWikiRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //wiki版本
    request.setWikiVersion(1L);
    GetMetaTableIntroWikiResponse res = client.getAcsResponse(request);
    //資料
    res.getData();           

GetMetaTableLineage

  • 擷取odps表血緣
GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    request.setDataSourceType("odps");
    //方向,up:上遊 down:下遊
    request.setDirection("up");
    //每頁大小
    request.setPageSize(10);
    //下一頁的起始key,null的時候從最開始查詢
    String nextPrimaryKey = null;
    //是否有下一頁
    Boolean hasNext = false;

    List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();

    do {
        //分頁起始key,傳回資料中擷取
        request.setNextPrimaryKey(nextPrimaryKey);

        GetMetaTableLineageResponse res = client.getAcsResponse(request);
        List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
        dataEntityListItems.addAll(items);
        //是否有下一頁
        hasNext = res.getData().getHasNext();
        //下一頁起始key
        nextPrimaryKey = res.getData().getNextPrimaryKey();

    } while(hasNext);           
  • 擷取emr表血緣
GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");
    //下一次請求開始id,參考 GetMetaTableLineageResponse.getData().getNextPrimaryKey()
    request.setDirection("up");
    request.setPageSize(10);
    //下一頁的起始key,null的時候從最開始查詢
    String nextPrimaryKey = null;
    //是否有下一頁
    Boolean hasNext = false;

    List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();

    do {
        //分頁起始key,傳回資料中擷取
        request.setNextPrimaryKey(nextPrimaryKey);

        GetMetaTableLineageResponse res = client.getAcsResponse(request);
        List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
        dataEntityListItems.addAll(items);
        //是否有下一頁
        hasNext = res.getData().getHasNext();
        //下一頁起始key
        nextPrimaryKey = res.getData().getNextPrimaryKey();

    } while(hasNext);           

GetMetaColumnLineage

  • 擷取odps字段血緣
GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
    //column guid,格式 odps.{projectName}.{tableName}.{columnName}
    request.setColumnGuid("odps.project1.t1");
    request.setDataSourceType("odps");
    //方向,up:上遊 down:下遊
    request.setDirection("down");
    //分頁
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaColumnLineageResponse res = client.getAcsResponse(request);
    //資料總條數
    res.getData().getTotalCount();
    //資料
    res.getData().getDataEntityList();           
  • 擷取emr字段血緣
GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
    //資料類型
    request.setDataSourceType("emr");
    //emr叢集id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 資料庫名稱
    request.setDatabaseName("default");
    //表名稱
    request.setTableName("table1");
    request.setDirection("up");
    request.setColumnName("eid");
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaColumnLineageResponse res = client.getAcsResponse(request);
    //資料總條數
    res.getData().getTotalCount();
    //資料
    res.getData().getDataEntityList();           

CreateMetaCategory

  • 建立類目
CreateMetaCategoryRequest request = new CreateMetaCategoryRequest();
    //類目名稱
    request.setName("test1");
    //注釋
    request.setComment("test");
    //父類目id,0表示沒有父類目
    request.setParentId(0L);

    CreateMetaCategoryResponse res = client.getAcsResponse(request);
    //生成的類目id
    res.getData().getCategoryId()           

UpdateMetaCategory

  • 更新類目
UpdateMetaCategoryRequest request = new UpdateMetaCategoryRequest();
    request.setCategoryId(53664L);
    request.setName("t93233");
    request.setComment("the com");
    UpdateMetaCategoryResponse res = client.getAcsResponse(request);
    //是否成功
    res.getData();           

DeleteMetaCategory

  • 删除類目
DeleteMetaCategoryRequest request = new DeleteMetaCategoryRequest();
    request.setCategoryId(53664L);
    DeleteMetaCategoryResponse res = client.getAcsResponse(request);
    //是否成功
    res.getData();           

GetMetaCategory

  • 擷取子類目清單
GetMetaCategoryRequest request = new GetMetaCategoryRequest();
    //父類目id
    request.setParentCategoryId(0L);
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaCategoryResponse res = client.getAcsResponse(request);
    //總資料
    res.getData().getTotalCount();
    //資料清單
    res.getData().getDataEntityList();           

GetMetaTableListByCategory

  • 擷取類目關聯表
GetMetaTableListByCategoryRequest request = new GetMetaTableListByCategoryRequest();
    request.setCategoryId(53664L);
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableListByCategoryResponse res = client.getAcsResponse(request);           

AddToMetaCategory

  • 增加類目、表關聯
AddToMetaCategoryRequest request = new AddToMetaCategoryRequest();
    request.setCategoryId(53664L);
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");

    AddToMetaCategoryResponse res = client.getAcsResponse(request);           

DeleteFromMetaCategory

  • 删除類目、表關聯
DeleteFromMetaCategoryRequest request = new DeleteFromMetaCategoryRequest();
    request.setCategoryId(53642L);
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    DeleteFromMetaCategoryResponse res = client.getAcsResponse(request);