Closed8
Delta Lake with UniForm の実験記録
nakazaxDelta Lake with UniForm のテーブルと通常の Delta Lake テーブルのフォルダ構造を比較する
nakazaxUnity Catalog 有効な Azure Databricks クラスターでカタログとスキーマを作成
-- Create catalog and schema for demo
CREATE CATALOG IF NOT EXISTS demo_catalog;
CREATE SCHEMA IF NOT EXISTS demo_catalog.demo_schema;
USE CATALOG demo_catalog;
USE SCHEMA demo_schema;
通常のテーブルを作成
-- Create normal delta table
CREATE TABLE IF NOT EXISTS dummy_users (ID STRING, Name STRING, Age INT);
ADLS Gen2 フォルダ
crc / json ファイル
_delta_log/00000000000000000000.crc
{
"txnId": "37412502-3f91-4990-99e3-0b556454dac8",
"tableSizeBytes": 0,
"numFiles": 0,
"numMetadata": 1,
"numProtocol": 1,
"setTransactions": [],
"domainMetadata": [],
"metadata": {
"id": "02e2c8d0-87b1-48b5-8095-9a5e402e8c50",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}",
"partitionColumns": [],
"configuration": {},
"createdTime": 1689053535940
},
"protocol": {
"minReaderVersion": 1,
"minWriterVersion": 2
},
"histogramOpt": {
"sortedBinBoundaries": [
0,
8192,
16384,
32768,
65536,
131072,
262144,
524288,
1048576,
2097152,
4194304,
8388608,
12582912,
16777216,
20971520,
25165824,
29360128,
33554432,
37748736,
41943040,
50331648,
58720256,
67108864,
75497472,
83886080,
92274688,
100663296,
109051904,
117440512,
125829120,
130023424,
134217728,
138412032,
142606336,
146800640,
150994944,
167772160,
184549376,
201326592,
218103808,
234881024,
251658240,
268435456,
285212672,
301989888,
318767104,
335544320,
352321536,
369098752,
385875968,
402653184,
419430400,
436207616,
452984832,
469762048,
486539264,
503316480,
520093696,
536870912,
553648128,
570425344,
587202560,
603979776,
671088640,
738197504,
805306368,
872415232,
939524096,
1006632960,
1073741824,
1140850688,
1207959552,
1275068416,
1342177280,
1409286144,
1476395008,
1610612736,
1744830464,
1879048192,
2013265920,
2147483648,
2415919104,
2684354560,
2952790016,
3221225472,
3489660928,
3758096384,
4026531840,
4294967296,
8589934592,
17179869184,
34359738368,
68719476736,
137438953472,
274877906944
],
"fileCounts": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"totalBytes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
"allFiles": []
}
_delta_log/00000000000000000000.json
{
"commitInfo": {
"timestamp": 1689053536048,
"userId": "8191114485960403",
"userName": "demo1@nakazax.onmicrosoft.com",
"operation": "CREATE TABLE",
"operationParameters": {
"partitionBy": "[]",
"description": null,
"isManaged": "true",
"properties": "{}",
"statsOnLoad": false
},
"notebook": {
"notebookId": "3228319588599460"
},
"clusterId": "0711-041515-7qh40oib",
"isolationLevel": "WriteSerializable",
"isBlindAppend": true,
"operationMetrics": {},
"engineInfo": "Databricks-Runtime/13.2.x-scala2.12",
"txnId": "37412502-3f91-4990-99e3-0b556454dac8"
}
}
{
"metaData": {
"id": "02e2c8d0-87b1-48b5-8095-9a5e402e8c50",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}",
"partitionColumns": [],
"configuration": {},
"createdTime": 1689053535940
}
}
{
"protocol": {
"minReaderVersion": 1,
"minWriterVersion": 2
}
}
通常のテーブルにダミーデータを挿入
-- Insert dummy data into normal delta table
INSERT INTO
dummy_users (ID, Name, Age)
VALUES
('1', 'Alice', 23),
('2', 'Bob', 27),
('3', 'Charlie', 22),
('4', 'Dave', 25),
('5', 'Eve', 24)
ADLS Gen2 フォルダ
crc / json ファイル
json
{
"txnId": "5762a113-e023-45bc-a39c-116c12303981",
"tableSizeBytes": 1071,
"numFiles": 1,
"numMetadata": 1,
"numProtocol": 1,
"setTransactions": [],
"domainMetadata": [],
"metadata": {
"id": "02e2c8d0-87b1-48b5-8095-9a5e402e8c50",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}",
"partitionColumns": [],
"configuration": {},
"createdTime": 1689053535940
},
"protocol": {
"minReaderVersion": 1,
"minWriterVersion": 2
},
"histogramOpt": {
"sortedBinBoundaries": [
0,
8192,
16384,
32768,
65536,
131072,
262144,
524288,
1048576,
2097152,
4194304,
8388608,
12582912,
16777216,
20971520,
25165824,
29360128,
33554432,
37748736,
41943040,
50331648,
58720256,
67108864,
75497472,
83886080,
92274688,
100663296,
109051904,
117440512,
125829120,
130023424,
134217728,
138412032,
142606336,
146800640,
150994944,
167772160,
184549376,
201326592,
218103808,
234881024,
251658240,
268435456,
285212672,
301989888,
318767104,
335544320,
352321536,
369098752,
385875968,
402653184,
419430400,
436207616,
452984832,
469762048,
486539264,
503316480,
520093696,
536870912,
553648128,
570425344,
587202560,
603979776,
671088640,
738197504,
805306368,
872415232,
939524096,
1006632960,
1073741824,
1140850688,
1207959552,
1275068416,
1342177280,
1409286144,
1476395008,
1610612736,
1744830464,
1879048192,
2013265920,
2147483648,
2415919104,
2684354560,
2952790016,
3221225472,
3489660928,
3758096384,
4026531840,
4294967296,
8589934592,
17179869184,
34359738368,
68719476736,
137438953472,
274877906944
],
"fileCounts": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"totalBytes": [
1071,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
"allFiles": [
{
"path": "part-00000-306daa0b-ec70-4f4c-856e-3cf9841c8d4f-c000.snappy.parquet",
"partitionValues": {},
"size": 1071,
"modificationTime": 1689053564000,
"dataChange": false,
"stats": "{\"numRecords\":5,\"minValues\":{\"ID\":\"1\",\"Name\":\"Alice\",\"Age\":22},\"maxValues\":{\"ID\":\"5\",\"Name\":\"Eve\",\"Age\":27},\"nullCount\":{\"ID\":0,\"Name\":0,\"Age\":0}}",
"tags": {
"INSERTION_TIME": "1689053564000000",
"MIN_INSERTION_TIME": "1689053564000000",
"MAX_INSERTION_TIME": "1689053564000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
]
}
_delta_log/00000000000000000001.json
{
"commitInfo": {
"timestamp": 1689053564132,
"userId": "8191114485960403",
"userName": "demo1@nakazax.onmicrosoft.com",
"operation": "WRITE",
"operationParameters": {
"mode": "Append",
"statsOnLoad": false,
"partitionBy": "[]"
},
"notebook": {
"notebookId": "3228319588599460"
},
"clusterId": "0711-041515-7qh40oib",
"readVersion": 0,
"isolationLevel": "WriteSerializable",
"isBlindAppend": true,
"operationMetrics": {
"numFiles": "1",
"numOutputRows": "5",
"numOutputBytes": "1071"
},
"tags": {
"restoresDeletedRows": "false"
},
"engineInfo": "Databricks-Runtime/13.2.x-scala2.12",
"txnId": "5762a113-e023-45bc-a39c-116c12303981"
}
}
{
"add": {
"path": "part-00000-306daa0b-ec70-4f4c-856e-3cf9841c8d4f-c000.snappy.parquet",
"partitionValues": {},
"size": 1071,
"modificationTime": 1689053564000,
"dataChange": true,
"stats": "{\"numRecords\":5,\"minValues\":{\"ID\":\"1\",\"Name\":\"Alice\",\"Age\":22},\"maxValues\":{\"ID\":\"5\",\"Name\":\"Eve\",\"Age\":27},\"nullCount\":{\"ID\":0,\"Name\":0,\"Age\":0}}",
"tags": {
"INSERTION_TIME": "1689053564000000",
"MIN_INSERTION_TIME": "1689053564000000",
"MAX_INSERTION_TIME": "1689053564000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
}
通常のテーブルにダミーデータをさらに挿入
-- Insert more dummy data into normal delta table
INSERT INTO demo_catalog.demo_schema.dummy_users (ID, Name, Age)
VALUES
('6', 'Frank', 28),
('7', 'Grace', 30),
('8', 'Helen', 26),
('9', 'Ivan', 31),
('10', 'Jack', 29)
ADLS Gen2 フォルダ
crc / json ファイル
_delta_log/00000000000000000002.log
{
"txnId": "d2ebdae5-9723-42c7-8be4-2dd443aee791",
"tableSizeBytes": 2147,
"numFiles": 2,
"numMetadata": 1,
"numProtocol": 1,
"setTransactions": [],
"domainMetadata": [],
"metadata": {
"id": "02e2c8d0-87b1-48b5-8095-9a5e402e8c50",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}",
"partitionColumns": [],
"configuration": {},
"createdTime": 1689053535940
},
"protocol": {
"minReaderVersion": 1,
"minWriterVersion": 2
},
"histogramOpt": {
"sortedBinBoundaries": [
0,
8192,
16384,
32768,
65536,
131072,
262144,
524288,
1048576,
2097152,
4194304,
8388608,
12582912,
16777216,
20971520,
25165824,
29360128,
33554432,
37748736,
41943040,
50331648,
58720256,
67108864,
75497472,
83886080,
92274688,
100663296,
109051904,
117440512,
125829120,
130023424,
134217728,
138412032,
142606336,
146800640,
150994944,
167772160,
184549376,
201326592,
218103808,
234881024,
251658240,
268435456,
285212672,
301989888,
318767104,
335544320,
352321536,
369098752,
385875968,
402653184,
419430400,
436207616,
452984832,
469762048,
486539264,
503316480,
520093696,
536870912,
553648128,
570425344,
587202560,
603979776,
671088640,
738197504,
805306368,
872415232,
939524096,
1006632960,
1073741824,
1140850688,
1207959552,
1275068416,
1342177280,
1409286144,
1476395008,
1610612736,
1744830464,
1879048192,
2013265920,
2147483648,
2415919104,
2684354560,
2952790016,
3221225472,
3489660928,
3758096384,
4026531840,
4294967296,
8589934592,
17179869184,
34359738368,
68719476736,
137438953472,
274877906944
],
"fileCounts": [
2,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"totalBytes": [
2147,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
"allFiles": [
{
"path": "part-00000-7acefef3-58e5-4a10-90a0-02c0136179d7-c000.snappy.parquet",
"partitionValues": {},
"size": 1076,
"modificationTime": 1689053598000,
"dataChange": false,
"stats": "{\"numRecords\":5,\"minValues\":{\"ID\":\"10\",\"Name\":\"Frank\",\"Age\":26},\"maxValues\":{\"ID\":\"9\",\"Name\":\"Jack\",\"Age\":31},\"nullCount\":{\"ID\":0,\"Name\":0,\"Age\":0}}",
"tags": {
"INSERTION_TIME": "1689053598000000",
"MIN_INSERTION_TIME": "1689053598000000",
"MAX_INSERTION_TIME": "1689053598000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
},
{
"path": "part-00000-306daa0b-ec70-4f4c-856e-3cf9841c8d4f-c000.snappy.parquet",
"partitionValues": {},
"size": 1071,
"modificationTime": 1689053564000,
"dataChange": false,
"stats": "{\"numRecords\":5,\"minValues\":{\"ID\":\"1\",\"Name\":\"Alice\",\"Age\":22},\"maxValues\":{\"ID\":\"5\",\"Name\":\"Eve\",\"Age\":27},\"nullCount\":{\"ID\":0,\"Name\":0,\"Age\":0}}",
"tags": {
"INSERTION_TIME": "1689053564000000",
"MIN_INSERTION_TIME": "1689053564000000",
"MAX_INSERTION_TIME": "1689053564000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
]
}
_delta_log/00000000000000000002.json
{
"commitInfo": {
"timestamp": 1689053598588,
"userId": "8191114485960403",
"userName": "demo1@nakazax.onmicrosoft.com",
"operation": "WRITE",
"operationParameters": {
"mode": "Append",
"statsOnLoad": false,
"partitionBy": "[]"
},
"notebook": {
"notebookId": "3228319588599460"
},
"clusterId": "0711-041515-7qh40oib",
"readVersion": 1,
"isolationLevel": "WriteSerializable",
"isBlindAppend": true,
"operationMetrics": {
"numFiles": "1",
"numOutputRows": "5",
"numOutputBytes": "1076"
},
"tags": {
"restoresDeletedRows": "false"
},
"engineInfo": "Databricks-Runtime/13.2.x-scala2.12",
"txnId": "d2ebdae5-9723-42c7-8be4-2dd443aee791"
}
}
{
"add": {
"path": "part-00000-7acefef3-58e5-4a10-90a0-02c0136179d7-c000.snappy.parquet",
"partitionValues": {},
"size": 1076,
"modificationTime": 1689053598000,
"dataChange": true,
"stats": "{\"numRecords\":5,\"minValues\":{\"ID\":\"10\",\"Name\":\"Frank\",\"Age\":26},\"maxValues\":{\"ID\":\"9\",\"Name\":\"Jack\",\"Age\":31},\"nullCount\":{\"ID\":0,\"Name\":0,\"Age\":0}}",
"tags": {
"INSERTION_TIME": "1689053598000000",
"MIN_INSERTION_TIME": "1689053598000000",
"MAX_INSERTION_TIME": "1689053598000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
}
UniForm テーブルを作成
-- Create delta table with UniForm
CREATE TABLE IF NOT EXISTS dummy_users_with_uniform (ID STRING, Name STRING, Age INT) TBLPROPERTIES(
'delta.universalFormat.enabledFormats' = 'iceberg'
);
ADLS Gen2 フォルダ
crc / json / metadata
_delta_log/00000000000000000000.crc
{
"txnId": "975841bd-5f6d-49b1-8e20-20eb8be37e02",
"tableSizeBytes": 0,
"numFiles": 0,
"numMetadata": 1,
"numProtocol": 1,
"setTransactions": [],
"domainMetadata": [],
"metadata": {
"id": "ac4bdba2-659c-4c55-bb78-abc7d0cb6165",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\"}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-a4528d83-820b-41f1-a40a-23856259c7e2\"}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\"}}]}",
"partitionColumns": [],
"configuration": {
"delta.universalFormat.enabledFormats": "iceberg",
"delta.enableIcebergCompatV1": "true",
"delta.columnMapping.mode": "name",
"delta.columnMapping.maxColumnId": "3"
},
"createdTime": 1689053639608
},
"protocol": {
"minReaderVersion": 3,
"minWriterVersion": 7,
"readerFeatures": [
"columnMapping"
],
"writerFeatures": [
"icebergCompatV1",
"appendOnly",
"invariants",
"columnMapping"
]
},
"histogramOpt": {
"sortedBinBoundaries": [
0,
8192,
16384,
32768,
65536,
131072,
262144,
524288,
1048576,
2097152,
4194304,
8388608,
12582912,
16777216,
20971520,
25165824,
29360128,
33554432,
37748736,
41943040,
50331648,
58720256,
67108864,
75497472,
83886080,
92274688,
100663296,
109051904,
117440512,
125829120,
130023424,
134217728,
138412032,
142606336,
146800640,
150994944,
167772160,
184549376,
201326592,
218103808,
234881024,
251658240,
268435456,
285212672,
301989888,
318767104,
335544320,
352321536,
369098752,
385875968,
402653184,
419430400,
436207616,
452984832,
469762048,
486539264,
503316480,
520093696,
536870912,
553648128,
570425344,
587202560,
603979776,
671088640,
738197504,
805306368,
872415232,
939524096,
1006632960,
1073741824,
1140850688,
1207959552,
1275068416,
1342177280,
1409286144,
1476395008,
1610612736,
1744830464,
1879048192,
2013265920,
2147483648,
2415919104,
2684354560,
2952790016,
3221225472,
3489660928,
3758096384,
4026531840,
4294967296,
8589934592,
17179869184,
34359738368,
68719476736,
137438953472,
274877906944
],
"fileCounts": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"totalBytes": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
"allFiles": []
}
_delta_log/00000000000000000000.json
{
"commitInfo": {
"timestamp": 1689053639740,
"userId": "8191114485960403",
"userName": "demo1@nakazax.onmicrosoft.com",
"operation": "CREATE TABLE",
"operationParameters": {
"partitionBy": "[]",
"description": null,
"isManaged": "true",
"properties": "{\"delta.universalFormat.enabledFormats\":\"iceberg\"}",
"statsOnLoad": false
},
"notebook": {
"notebookId": "3228319588599460"
},
"clusterId": "0711-041515-7qh40oib",
"isolationLevel": "WriteSerializable",
"isBlindAppend": true,
"operationMetrics": {},
"engineInfo": "Databricks-Runtime/13.2.x-scala2.12",
"txnId": "975841bd-5f6d-49b1-8e20-20eb8be37e02"
}
}
{
"metaData": {
"id": "ac4bdba2-659c-4c55-bb78-abc7d0cb6165",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\"}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-a4528d83-820b-41f1-a40a-23856259c7e2\"}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\"}}]}",
"partitionColumns": [],
"configuration": {
"delta.universalFormat.enabledFormats": "iceberg",
"delta.enableIcebergCompatV1": "true",
"delta.columnMapping.mode": "name",
"delta.columnMapping.maxColumnId": "3"
},
"createdTime": 1689053639608
}
}
{
"protocol": {
"minReaderVersion": 3,
"minWriterVersion": 7,
"readerFeatures": [
"columnMapping"
],
"writerFeatures": [
"icebergCompatV1",
"appendOnly",
"invariants",
"columnMapping"
]
}
}
metadata/00000-082e47af-9f25-46f1-9277-d2c7e91b7617.metadata.json
{
"format-version" : 1,
"table-uuid" : "6c1e611c-c38f-4212-8ce6-54c7acd23841",
"location" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2",
"last-updated-ms" : 1689053642078,
"last-column-id" : 3,
"schema" : {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "ID",
"required" : false,
"type" : "string"
}, {
"id" : 2,
"name" : "Name",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "Age",
"required" : false,
"type" : "int"
} ]
},
"current-schema-id" : 0,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "ID",
"required" : false,
"type" : "string"
}, {
"id" : 2,
"name" : "Name",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "Age",
"required" : false,
"type" : "int"
} ]
} ],
"partition-spec" : [ ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"delta-timestamp" : "1689053639000",
"delta-version" : "0"
},
"current-snapshot-id" : -1,
"refs" : { },
"snapshots" : [ ],
"statistics" : [ ],
"snapshot-log" : [ ],
"metadata-log" : [ ]
}
UniForm テーブルにダミーデータを挿入
-- Insert dummy data into delta table with UniForm
INSERT INTO
dummy_users_with_uniform (ID, Name, Age)
VALUES
('1', 'Alice', 23),
('2', 'Bob', 27),
('3', 'Charlie', 22),
('4', 'Dave', 25),
('5', 'Eve', 24)
ADLS Gen2 フォルダ
SU というフォルダにデータファイルがある
_delta_log は通常と変わらず
metadata と avro ファイルが増えている
crc / json / metadata / avro
_delta_log/00000000000000000001.crc
{
"txnId": "4c599cb6-102b-473f-a96e-308f6df4fa49",
"tableSizeBytes": 1785,
"numFiles": 1,
"numMetadata": 1,
"numProtocol": 1,
"setTransactions": [],
"domainMetadata": [],
"metadata": {
"id": "ac4bdba2-659c-4c55-bb78-abc7d0cb6165",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\"}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-a4528d83-820b-41f1-a40a-23856259c7e2\"}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\"}}]}",
"partitionColumns": [],
"configuration": {
"delta.universalFormat.enabledFormats": "iceberg",
"delta.enableIcebergCompatV1": "true",
"delta.columnMapping.mode": "name",
"delta.columnMapping.maxColumnId": "3"
},
"createdTime": 1689053639608
},
"protocol": {
"minReaderVersion": 3,
"minWriterVersion": 7,
"readerFeatures": [
"columnMapping"
],
"writerFeatures": [
"icebergCompatV1",
"appendOnly",
"invariants",
"columnMapping"
]
},
"histogramOpt": {
"sortedBinBoundaries": [
0,
8192,
16384,
32768,
65536,
131072,
262144,
524288,
1048576,
2097152,
4194304,
8388608,
12582912,
16777216,
20971520,
25165824,
29360128,
33554432,
37748736,
41943040,
50331648,
58720256,
67108864,
75497472,
83886080,
92274688,
100663296,
109051904,
117440512,
125829120,
130023424,
134217728,
138412032,
142606336,
146800640,
150994944,
167772160,
184549376,
201326592,
218103808,
234881024,
251658240,
268435456,
285212672,
301989888,
318767104,
335544320,
352321536,
369098752,
385875968,
402653184,
419430400,
436207616,
452984832,
469762048,
486539264,
503316480,
520093696,
536870912,
553648128,
570425344,
587202560,
603979776,
671088640,
738197504,
805306368,
872415232,
939524096,
1006632960,
1073741824,
1140850688,
1207959552,
1275068416,
1342177280,
1409286144,
1476395008,
1610612736,
1744830464,
1879048192,
2013265920,
2147483648,
2415919104,
2684354560,
2952790016,
3221225472,
3489660928,
3758096384,
4026531840,
4294967296,
8589934592,
17179869184,
34359738368,
68719476736,
137438953472,
274877906944
],
"fileCounts": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"totalBytes": [
1785,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
"allFiles": [
{
"path": "SU/part-00000-9ff7dc33-466f-4614-b39b-bd36316e1d53-c000.snappy.parquet",
"partitionValues": {},
"size": 1785,
"modificationTime": 1689053679000,
"dataChange": false,
"stats": "{\"numRecords\":5,\"minValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"1\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Alice\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":22},\"maxValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"5\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Eve\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":27},\"nullCount\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":0,\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":0,\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":0}}",
"tags": {
"INSERTION_TIME": "1689053679000000",
"MIN_INSERTION_TIME": "1689053679000000",
"MAX_INSERTION_TIME": "1689053679000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
]
}
_delta_log/00000000000000000001.json
{
"commitInfo": {
"timestamp": 1689053679221,
"userId": "8191114485960403",
"userName": "demo1@nakazax.onmicrosoft.com",
"operation": "WRITE",
"operationParameters": {
"mode": "Append",
"statsOnLoad": false,
"partitionBy": "[]"
},
"notebook": {
"notebookId": "3228319588599460"
},
"clusterId": "0711-041515-7qh40oib",
"readVersion": 0,
"isolationLevel": "WriteSerializable",
"isBlindAppend": true,
"operationMetrics": {
"numFiles": "1",
"numOutputRows": "5",
"numOutputBytes": "1785"
},
"tags": {
"restoresDeletedRows": "false"
},
"engineInfo": "Databricks-Runtime/13.2.x-scala2.12",
"txnId": "4c599cb6-102b-473f-a96e-308f6df4fa49"
}
}
{
"add": {
"path": "SU/part-00000-9ff7dc33-466f-4614-b39b-bd36316e1d53-c000.snappy.parquet",
"partitionValues": {},
"size": 1785,
"modificationTime": 1689053679000,
"dataChange": true,
"stats": "{\"numRecords\":5,\"minValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"1\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Alice\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":22},\"maxValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"5\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Eve\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":27},\"nullCount\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":0,\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":0,\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":0}}",
"tags": {
"INSERTION_TIME": "1689053679000000",
"MIN_INSERTION_TIME": "1689053679000000",
"MAX_INSERTION_TIME": "1689053679000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
}
metadata/00001-97f12be2-9549-4789-b4cb-9d9075d2501f.metadata.json
{
"format-version" : 1,
"table-uuid" : "6c1e611c-c38f-4212-8ce6-54c7acd23841",
"location" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2",
"last-updated-ms" : 1689053681002,
"last-column-id" : 3,
"schema" : {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "ID",
"required" : false,
"type" : "string"
}, {
"id" : 2,
"name" : "Name",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "Age",
"required" : false,
"type" : "int"
} ]
},
"current-schema-id" : 0,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "ID",
"required" : false,
"type" : "string"
}, {
"id" : 2,
"name" : "Name",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "Age",
"required" : false,
"type" : "int"
} ]
} ],
"partition-spec" : [ ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"delta-timestamp" : "1689053679000",
"delta-version" : "1"
},
"current-snapshot-id" : 3332464395990706880,
"refs" : {
"main" : {
"snapshot-id" : 3332464395990706880,
"type" : "branch"
}
},
"snapshots" : [ {
"snapshot-id" : 3332464395990706880,
"timestamp-ms" : 1689053680973,
"summary" : {
"operation" : "append",
"added-data-files" : "1",
"added-records" : "5",
"added-files-size" : "1785",
"changed-partition-count" : "1",
"total-records" : "5",
"total-files-size" : "1785",
"total-data-files" : "1",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2/metadata/snap-3332464395990706880-1-b27de1ea-c872-4074-9699-ca350b5f6403.avro",
"schema-id" : 0
} ],
"statistics" : [ ],
"snapshot-log" : [ {
"timestamp-ms" : 1689053680973,
"snapshot-id" : 3332464395990706880
} ],
"metadata-log" : [ {
"timestamp-ms" : 1689053642078,
"metadata-file" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2/metadata/00000-082e47af-9f25-46f1-9277-d2c7e91b7617.metadata.json"
} ]
}
UniForm テーブルにさらにダミーデータを挿入
-- Insert more dummy data into delta table with UniForm
INSERT INTO demo_catalog.demo_schema.dummy_users_with_uniform (ID, Name, Age)
VALUES
('6', 'Frank', 28),
('7', 'Grace', 30),
('8', 'Helen', 26),
('9', 'Ivan', 31),
('10', 'Jack', 29)
ADLS Gen2 フォルダ
pB が増えている
pB にデータファイルが格納されている
_delta_log は通常と変わらず
metadata の JSON と Avro が増えている
crc / json / metadata / avro
00000000000000000002.crc
{
"txnId": "ccaa9e74-8e1d-4bd1-a595-3df70b94c4c7",
"tableSizeBytes": 3575,
"numFiles": 2,
"numMetadata": 1,
"numProtocol": 1,
"setTransactions": [],
"domainMetadata": [],
"metadata": {
"id": "ac4bdba2-659c-4c55-bb78-abc7d0cb6165",
"format": {
"provider": "parquet",
"options": {}
},
"schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\"}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-a4528d83-820b-41f1-a40a-23856259c7e2\"}},{\"name\":\"Age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\"}}]}",
"partitionColumns": [],
"configuration": {
"delta.universalFormat.enabledFormats": "iceberg",
"delta.enableIcebergCompatV1": "true",
"delta.columnMapping.mode": "name",
"delta.columnMapping.maxColumnId": "3"
},
"createdTime": 1689053639608
},
"protocol": {
"minReaderVersion": 3,
"minWriterVersion": 7,
"readerFeatures": [
"columnMapping"
],
"writerFeatures": [
"icebergCompatV1",
"appendOnly",
"invariants",
"columnMapping"
]
},
"histogramOpt": {
"sortedBinBoundaries": [
0,
8192,
16384,
32768,
65536,
131072,
262144,
524288,
1048576,
2097152,
4194304,
8388608,
12582912,
16777216,
20971520,
25165824,
29360128,
33554432,
37748736,
41943040,
50331648,
58720256,
67108864,
75497472,
83886080,
92274688,
100663296,
109051904,
117440512,
125829120,
130023424,
134217728,
138412032,
142606336,
146800640,
150994944,
167772160,
184549376,
201326592,
218103808,
234881024,
251658240,
268435456,
285212672,
301989888,
318767104,
335544320,
352321536,
369098752,
385875968,
402653184,
419430400,
436207616,
452984832,
469762048,
486539264,
503316480,
520093696,
536870912,
553648128,
570425344,
587202560,
603979776,
671088640,
738197504,
805306368,
872415232,
939524096,
1006632960,
1073741824,
1140850688,
1207959552,
1275068416,
1342177280,
1409286144,
1476395008,
1610612736,
1744830464,
1879048192,
2013265920,
2147483648,
2415919104,
2684354560,
2952790016,
3221225472,
3489660928,
3758096384,
4026531840,
4294967296,
8589934592,
17179869184,
34359738368,
68719476736,
137438953472,
274877906944
],
"fileCounts": [
2,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"totalBytes": [
3575,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
"allFiles": [
{
"path": "pB/part-00000-cbdb3ead-99b4-4fb0-88b6-eaf45a74f335-c000.snappy.parquet",
"partitionValues": {},
"size": 1790,
"modificationTime": 1689053763000,
"dataChange": false,
"stats": "{\"numRecords\":5,\"minValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"10\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Frank\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":26},\"maxValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"9\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Jack\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":31},\"nullCount\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":0,\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":0,\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":0}}",
"tags": {
"INSERTION_TIME": "1689053763000000",
"MIN_INSERTION_TIME": "1689053763000000",
"MAX_INSERTION_TIME": "1689053763000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
},
{
"path": "SU/part-00000-9ff7dc33-466f-4614-b39b-bd36316e1d53-c000.snappy.parquet",
"partitionValues": {},
"size": 1785,
"modificationTime": 1689053679000,
"dataChange": false,
"stats": "{\"numRecords\":5,\"minValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"1\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Alice\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":22},\"maxValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"5\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Eve\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":27},\"nullCount\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":0,\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":0,\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":0}}",
"tags": {
"INSERTION_TIME": "1689053679000000",
"MIN_INSERTION_TIME": "1689053679000000",
"MAX_INSERTION_TIME": "1689053679000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
]
}
00000000000000000002.json
{
"commitInfo": {
"timestamp": 1689053763351,
"userId": "8191114485960403",
"userName": "demo1@nakazax.onmicrosoft.com",
"operation": "WRITE",
"operationParameters": {
"mode": "Append",
"statsOnLoad": false,
"partitionBy": "[]"
},
"notebook": {
"notebookId": "3228319588599460"
},
"clusterId": "0711-041515-7qh40oib",
"readVersion": 1,
"isolationLevel": "WriteSerializable",
"isBlindAppend": true,
"operationMetrics": {
"numFiles": "1",
"numOutputRows": "5",
"numOutputBytes": "1790"
},
"tags": {
"restoresDeletedRows": "false"
},
"engineInfo": "Databricks-Runtime/13.2.x-scala2.12",
"txnId": "ccaa9e74-8e1d-4bd1-a595-3df70b94c4c7"
}
}
{
"add": {
"path": "pB/part-00000-cbdb3ead-99b4-4fb0-88b6-eaf45a74f335-c000.snappy.parquet",
"partitionValues": {},
"size": 1790,
"modificationTime": 1689053763000,
"dataChange": true,
"stats": "{\"numRecords\":5,\"minValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"10\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Frank\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":26},\"maxValues\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":\"9\",\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":\"Jack\",\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":31},\"nullCount\":{\"col-f0c5749f-1221-4263-a66b-31d47da5e90b\":0,\"col-a4528d83-820b-41f1-a40a-23856259c7e2\":0,\"col-4fbe4c9e-5352-434d-ae92-86c94c2400cb\":0}}",
"tags": {
"INSERTION_TIME": "1689053763000000",
"MIN_INSERTION_TIME": "1689053763000000",
"MAX_INSERTION_TIME": "1689053763000000",
"OPTIMIZE_TARGET_SIZE": "67108864"
}
}
}
metadata/00002-f5bae09e-3ac5-475e-912f-b7f668cf0c4d.metadata.json
{
"format-version" : 1,
"table-uuid" : "6c1e611c-c38f-4212-8ce6-54c7acd23841",
"location" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2",
"last-updated-ms" : 1689053765863,
"last-column-id" : 3,
"schema" : {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "ID",
"required" : false,
"type" : "string"
}, {
"id" : 2,
"name" : "Name",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "Age",
"required" : false,
"type" : "int"
} ]
},
"current-schema-id" : 0,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "ID",
"required" : false,
"type" : "string"
}, {
"id" : 2,
"name" : "Name",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "Age",
"required" : false,
"type" : "int"
} ]
} ],
"partition-spec" : [ ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"delta-timestamp" : "1689053763000",
"delta-version" : "2"
},
"current-snapshot-id" : 7716244493843165821,
"refs" : {
"main" : {
"snapshot-id" : 7716244493843165821,
"type" : "branch"
}
},
"snapshots" : [ {
"snapshot-id" : 3332464395990706880,
"timestamp-ms" : 1689053680973,
"summary" : {
"operation" : "append",
"added-data-files" : "1",
"added-records" : "5",
"added-files-size" : "1785",
"changed-partition-count" : "1",
"total-records" : "5",
"total-files-size" : "1785",
"total-data-files" : "1",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2/metadata/snap-3332464395990706880-1-b27de1ea-c872-4074-9699-ca350b5f6403.avro",
"schema-id" : 0
}, {
"snapshot-id" : 7716244493843165821,
"parent-snapshot-id" : 3332464395990706880,
"timestamp-ms" : 1689053765836,
"summary" : {
"operation" : "append",
"added-data-files" : "1",
"added-records" : "5",
"added-files-size" : "1790",
"changed-partition-count" : "1",
"total-records" : "10",
"total-files-size" : "3575",
"total-data-files" : "2",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2/metadata/snap-7716244493843165821-1-82cb65ff-34d0-444e-95a7-ae5dbc2f84df.avro",
"schema-id" : 0
} ],
"statistics" : [ ],
"snapshot-log" : [ {
"timestamp-ms" : 1689053680973,
"snapshot-id" : 3332464395990706880
}, {
"timestamp-ms" : 1689053765836,
"snapshot-id" : 7716244493843165821
} ],
"metadata-log" : [ {
"timestamp-ms" : 1689053642078,
"metadata-file" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2/metadata/00000-082e47af-9f25-46f1-9277-d2c7e91b7617.metadata.json"
}, {
"timestamp-ms" : 1689053681002,
"metadata-file" : "abfss://metastore@stnakadbucejp.dfs.core.windows.net/c1d6596c-507e-427d-a1ea-535b001d4d75/tables/f8c83899-5712-49be-8710-43a892c809d2/metadata/00001-97f12be2-9549-4789-b4cb-9d9075d2501f.metadata.json"
} ]
}
このスクラップは2023/07/11にクローズされました