以下demo内容已經較老,大家使用API建議移步官方API文檔: https://help.aliyun.com/document_detail/173915.html 有詳細的demo代碼和線上調試功能
DataWorks OpenAPI 中繼資料demo
Client初始化
String region = "cn-shanghai";
IClientProfile profile = DefaultProfile.getProfile(region, "xxx", "xxx");
DefaultProfile.addEndpoint(region,"dataworks-public", String.format("dataworks.%s.aliyuncs.com", region));
IAcsClient client = new DefaultAcsClient(profile);
GetMetaDBInfo
- 擷取odps引擎詳情
GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
//odps app guid,格式 odps.{projectName}
request.setAppGuid("odps.project1");
//類型為odps
request.setDataSourceType("odps");
GetMetaDBInfoResponse res = client.getAcsResponse(request);
GetMetaDBInfoResponse.Data data = res.getData();
- 擷取emr庫詳情
GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
request.setDataSourceType("emr");
//emr 叢集id
request.setClusterId("C-C206DE8CACxxxxx");
//emr db名稱
request.setDatabaseName("default");
GetMetaDBInfoResponse res = client.getAcsResponse(request);
GetMetaDBInfoResponse.Data data = res.getData();
ListMetaDB
- 擷取emr 資料庫清單
ListMetaDBRequest request = new ListMetaDBRequest();
//資料類型
request.setDataSourceType("emr");
//dataworks 項目id
request.setProjectId(110L);
request.setPageSize(1);
request.setPageNum(10);
ListMetaDBResponse res = client.getAcsResponse(request);
//總條數
res.getDatabaseInfo().getTotalCount();
//資料
res.getDatabaseInfo().getDbList();
GetMetaDBTableList
- 擷取odps引擎關聯表
GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
//app guid
request.setAppGuid("odps.project1");
request.setDataSourceType("odps");
//第1頁
request.setPageNumber(1);
//每頁大小
request.setPageSize(10);
GetMetaDBTableListResponse res = client.getAcsResponse(request);
//資料總大小
Long totalCount = res.getData().getTotalCount();
//目前頁資料清單
List<GetMetaDBTableListResponse.Data.TableEntityListItem> tableEntityListItems = res.getData().getTableEntityList();
- 擷取emr 資料庫下的表
GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CAC0xxxxx");
//emr 資料庫名稱
request.setDatabaseName("default");
GetMetaDBTableListResponse res = client.getAcsResponse(request);
//資料總大小
Long totalCount = res.getData().getTotalCount();
//目前頁資料清單
List<GetMetaDBTableListResponse.Data.TableEntityListItem> tableEntityListItems = res.getData().getTableEntityList();
CheckMetaTable
- 檢查odps表是否存在
CheckMetaTableRequest request = new CheckMetaTableRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//資源類型
request.setDataSourceType("odps");
CheckMetaTableResponse res = client.getAcsResponse(request);
//表是否存在
Boolean exists = res.getData();
- 檢查emr表是否存在
CheckMetaTableRequest request = new CheckMetaTableRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
CheckMetaTableResponse res = client.getAcsResponse(request);
CheckMetaPartition
- 檢查odps分區是否存在
CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//資源類型
request.setDataSourceType("odps");
//分區
request.setPartition("dt_time=20200419");
CheckMetaPartitionResponse res = client.getAcsResponse(request);
//分區是否存在
Boolean exists = res.getData();
- 檢查emr分區是否存在
CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
request.setPartition("ds=202001");
CheckMetaPartitionResponse res = client.getAcsResponse(request);
//分區是否存在
Boolean exists = res.getData();
SearchMetaTables
- 搜尋odps表
SearchMetaTablesRequest request = new SearchMetaTablesRequest();
//odps app guid,格式 odps.{projectName}
request.setAppGuid("odps.p1");
request.setDataSourceType("odps");
//表類型,0:table, 1:view 預設全部
request.setEntityType(0);
//關鍵字
request.setKeyword("test");
//分頁
request.setPageNumber(1);
request.setPageSize(10);
SearchMetaTablesResponse res = client.getAcsResponse(request);
//資料總條數
Long totalCount = res.getData().getTotalCount();
//目前頁表清單
res.getData().getDataEntityList();
- 搜尋emr表
SearchMetaTablesRequest request = new SearchMetaTablesRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
request.setKeyword("test");
request.setPageNumber(1);
request.setPageSize(10);
SearchMetaTablesResponse res = client.getAcsResponse(request);
//資料總條數
Long totalCount = res.getData().getTotalCount();
//目前頁表清單
res.getData().getDataEntityList();
GetMetaTableBasicInfo
- 擷取odps表基本資訊
GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//資源類型
request.setDataSourceType("odps");
GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
//表資料
GetMetaTableBasicInfoResponse.Data info = res.getData();
- 擷取emr表基礎資訊
GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
//表資料
GetMetaTableBasicInfoResponse.Data info = res.getData();
GetMetaTableColumn
- 擷取odps column清單
GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//資源類型
request.setDataSourceType("odps");
//分頁
request.setPageNum(1);
request.setPageSize(10);
GetMetaTableColumnResponse res = client.getAcsResponse(request);
//資料總條數
Long totalCount = res.getData().getTotalCount();
//目前頁資料清單
List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();
- 擷取emr表column資訊
GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
request.setPageNum(1);
request.setPageSize(10);
GetMetaTableColumnResponse res = client.getAcsResponse(request);
//資料總條數
Long totalCount = res.getData().getTotalCount();
//目前頁資料清單
List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();
GetMetaTablePartition
- 擷取odps表分區
GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//資源類型
request.setDataSourceType("odps");
//分頁
request.setPageNumber(1);
request.setPageSize(10);
GetMetaTablePartitionResponse res = client.getAcsResponse(request);
//資料總條數
res.getData().getTotalCount();
//分區清單
res.getData().getDataEntityList();
- 擷取emr表分區
GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
request.setPageNumber(1);
request.setPageSize(10);
GetMetaTablePartitionResponse res = client.getAcsResponse(request);
//資料總條數
res.getData().getTotalCount();
//分區清單
res.getData().getDataEntityList();
GetMetaTableOutput
- 擷取odps表輸出
GetMetaTableOutputRequest request = new GetMetaTableOutputRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//output開始時間
request.setStartDate("2020-05-27");
//output結束時間
request.setEndDate("2020-06-12");
request.setPageNumber(1);
request.setPageSize(10);
GetMetaTableOutputResponse res = client.getAcsResponse(request);
//資料總條數
res.getData().getTotalCount();
//輸出清單
res.getData().getDataEntityList();
GetMetaTableChangeLog
- 擷取odps表變更資訊
GetMetaTableChangeLogRequest request = new GetMetaTableChangeLogRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//開始時間,格式yyyy-MM-dd HH:mm:ss
request.setStartDate("2020-01-01 01:01:00");
//格式yyyy-MM-dd HH:mm:ss
request.setEndDate("2020-01-02 01:01:00");
/**
* 類型:
* CREATE_TABLE
* ALTER_TABLE
* DROP_TABLE
* ADD_PARTITION
* DROP_PARTITION
* ALTER_OWNER_ID
* ALTER_LIFE_CYCLE_TIME
*/
request.setChangeType("CREATE_TABLE");
//對象 類型 TABLE,PARTITION
request.setObjectType("TABLE");
//分頁
request.setPageNumber(1);
request.setPageSize(10);
GetMetaTableChangeLogResponse res = client.getAcsResponse(request);
//資料總條數
res.getData().getTotalCount();
//資料
res.getData().getDataEntityList();
GetMetaTableIntroWiki
- 擷取wiki
GetMetaTableIntroWikiRequest request = new GetMetaTableIntroWikiRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
//wiki版本
request.setWikiVersion(1L);
GetMetaTableIntroWikiResponse res = client.getAcsResponse(request);
//資料
res.getData();
GetMetaTableLineage
- 擷取odps表血緣
GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
request.setDataSourceType("odps");
//方向,up:上遊 down:下遊
request.setDirection("up");
//每頁大小
request.setPageSize(10);
//下一頁的起始key,null的時候從最開始查詢
String nextPrimaryKey = null;
//是否有下一頁
Boolean hasNext = false;
List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();
do {
//分頁起始key,傳回資料中擷取
request.setNextPrimaryKey(nextPrimaryKey);
GetMetaTableLineageResponse res = client.getAcsResponse(request);
List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
dataEntityListItems.addAll(items);
//是否有下一頁
hasNext = res.getData().getHasNext();
//下一頁起始key
nextPrimaryKey = res.getData().getNextPrimaryKey();
} while(hasNext);
- 擷取emr表血緣
GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
//下一次請求開始id,參考 GetMetaTableLineageResponse.getData().getNextPrimaryKey()
request.setDirection("up");
request.setPageSize(10);
//下一頁的起始key,null的時候從最開始查詢
String nextPrimaryKey = null;
//是否有下一頁
Boolean hasNext = false;
List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();
do {
//分頁起始key,傳回資料中擷取
request.setNextPrimaryKey(nextPrimaryKey);
GetMetaTableLineageResponse res = client.getAcsResponse(request);
List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
dataEntityListItems.addAll(items);
//是否有下一頁
hasNext = res.getData().getHasNext();
//下一頁起始key
nextPrimaryKey = res.getData().getNextPrimaryKey();
} while(hasNext);
GetMetaColumnLineage
- 擷取odps字段血緣
GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
//column guid,格式 odps.{projectName}.{tableName}.{columnName}
request.setColumnGuid("odps.project1.t1");
request.setDataSourceType("odps");
//方向,up:上遊 down:下遊
request.setDirection("down");
//分頁
request.setPageNum(1);
request.setPageSize(10);
GetMetaColumnLineageResponse res = client.getAcsResponse(request);
//資料總條數
res.getData().getTotalCount();
//資料
res.getData().getDataEntityList();
- 擷取emr字段血緣
GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
//資料類型
request.setDataSourceType("emr");
//emr叢集id
request.setClusterId("C-C206DE8CXXXXXXX");
//emr 資料庫名稱
request.setDatabaseName("default");
//表名稱
request.setTableName("table1");
request.setDirection("up");
request.setColumnName("eid");
request.setPageNum(1);
request.setPageSize(10);
GetMetaColumnLineageResponse res = client.getAcsResponse(request);
//資料總條數
res.getData().getTotalCount();
//資料
res.getData().getDataEntityList();
CreateMetaCategory
- 建立類目
CreateMetaCategoryRequest request = new CreateMetaCategoryRequest();
//類目名稱
request.setName("test1");
//注釋
request.setComment("test");
//父類目id,0表示沒有父類目
request.setParentId(0L);
CreateMetaCategoryResponse res = client.getAcsResponse(request);
//生成的類目id
res.getData().getCategoryId()
UpdateMetaCategory
- 更新類目
UpdateMetaCategoryRequest request = new UpdateMetaCategoryRequest();
request.setCategoryId(53664L);
request.setName("t93233");
request.setComment("the com");
UpdateMetaCategoryResponse res = client.getAcsResponse(request);
//是否成功
res.getData();
DeleteMetaCategory
- 删除類目
DeleteMetaCategoryRequest request = new DeleteMetaCategoryRequest();
request.setCategoryId(53664L);
DeleteMetaCategoryResponse res = client.getAcsResponse(request);
//是否成功
res.getData();
GetMetaCategory
- 擷取子類目清單
GetMetaCategoryRequest request = new GetMetaCategoryRequest();
//父類目id
request.setParentCategoryId(0L);
request.setPageNum(1);
request.setPageSize(10);
GetMetaCategoryResponse res = client.getAcsResponse(request);
//總資料
res.getData().getTotalCount();
//資料清單
res.getData().getDataEntityList();
GetMetaTableListByCategory
- 擷取類目關聯表
GetMetaTableListByCategoryRequest request = new GetMetaTableListByCategoryRequest();
request.setCategoryId(53664L);
request.setPageNumber(1);
request.setPageSize(10);
GetMetaTableListByCategoryResponse res = client.getAcsResponse(request);
AddToMetaCategory
- 增加類目、表關聯
AddToMetaCategoryRequest request = new AddToMetaCategoryRequest();
request.setCategoryId(53664L);
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
AddToMetaCategoryResponse res = client.getAcsResponse(request);
DeleteFromMetaCategory
- 删除類目、表關聯
DeleteFromMetaCategoryRequest request = new DeleteFromMetaCategoryRequest();
request.setCategoryId(53642L);
//odps table guid,格式odps.{projectName}.{tableName}
request.setTableGuid("odps.project1.t1");
DeleteFromMetaCategoryResponse res = client.getAcsResponse(request);