淘先锋技术网

首页 1 2 3 4 5 6 7

对象存储应用开发

实验 1 搭建对象存储应用开发环境
Maven依赖的POM文件：

    <dependencies>

        <dependency>
            <groupId>com.sequoiadb</groupId>
            <artifactId>sequoiadb-driver</artifactId>
            <version>3.4</version>
        </dependency>

        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.11.343</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>

        <dependency>
            <groupId>org.dom4j</groupId>
            <artifactId>dom4j</artifactId>
            <version>2.1.1</version>
        </dependency>

    </dependencies>

使用Java连接S3

    //Initialize the client and generate a connection to SequoiaS3. Here, the address and port of endPoint need to be modified to point to the address and port of SequoiaS3.
    AWSCredentials credentials = 
            new BasicAWSCredentials("ABCDEFGHIJKLMNOPQRST",
                "abcdefghijklmnopqrstuvwxyz0123456789ABCD");
    String endPoint = "http://127.0.0.1:8002";
    AwsClientBuilder.EndpointConfiguration endpointConfiguration = 
        new AwsClientBuilder.EndpointConfiguration(endPoint, null);
    //Create the S3 connection object
    s3 = AmazonS3ClientBuilder.standard()
        .withEndpointConfiguration(endpointConfiguration)
        .withCredentials(
        new AWSStaticCredentialsProvider(credentials)
        ).build();

实验 2 存储桶、对象、存储区域操作
存储桶的创建:

// Get the S3 connection and create a bucket
AmazonS3 s3 = this.getS3();
// Call the interface for creating a bucket
s3.createBucket(bucketName);

存储桶的查询:

// List all current buckets
AmazonS3 s3 = this.getS3();
List<Bucket> buckets = s3.listBuckets();
// Output the list length, which is the number of buckets
System.out.println("bucket number:" + buckets.size());
// Traverse to output the name and creation date of each bucket
for (int i = 0; i < buckets.size(); i++) {
    Bucket bucket = buckets.get(i);
    System.out.println("bucketname:" + bucket.getName() +
                       ", date:" + bucket.getCreationDate());
}

存储桶的删除:

// Get the S3 connection, then delete the bucket
AmazonS3 s3 = this.getS3();
s3.deleteBucket(bucketName);

对象的创建
AWS S3 提供的有对象的创建接口 putObject(String var1, String var2, String var3) ,该接口在 SequoiaS3 中同样适用，可以通过 S3 对象直接调用该接口，创建指定的对象。参数意义依次为存储桶名称、对象名称、要保存的内容。

// Get the S3 connection
AmazonS3 s3 = this.getS3();
// Create the object
s3.putObject(bucketName, objectName, content);

对象的查询
AWS S3 提供的有对象的创建接口 listObjects(String var1)，该接口在 SequoiaS3 中同样适用，可以通过 S3 对象直接调用该接口，查询指定存储桶内的对象。参数意义为要查询存储桶名称。

// Call a method to get the list of objects in the specified bucket
AmazonS3 s3 = this.getS3();
ObjectListing objectListing = s3.listObjects(bucketName);
List<S3ObjectSummary> objectList =  objectListing.getObjectSummaries();
//Output the object information
System.out.println("key count: "+objectList.size());
for (int i = 0; i < objectList.size(); i++){
    System.out.println("key " +i + ": "+objectList.get(i).getKey());
}

对象的下载
WS S3 提供的有对象的下载接口 getObject(String var1, String var2),该接口在SequoiaS3 中同样适用，可以通过 S3 对象直接调用该接口，获取指定对象。参数意义依次为存储桶名称，对象名称。

AmazonS3 s3 = this.getS3();
// Get a reference to the specified object
S3Object result = s3.getObject(bucketName,objectName);
S3ObjectInputStream s3is = result.getObjectContent();
File file = new File(objectName);
//Print the path of file
System.out.println("file path:" + file.getAbsolutePath());
//Get file output stream to write file to disk
FileOutputStream fos = new FileOutputStream(file);
byte[] read_buf = new byte[1024 * 1024];
int read_len;
while ((read_len = s3is.read(read_buf)) > 0) {
    fos.write(read_buf, 0, read_len);
}
s3is.close();
fos.close();

对象的删除
AWS S3 提供的有对象的删除接口 deleteObject(String var1, String var2)，该接口在SequoiaS3 中同样适用，可以通过 S3 对象直接调用该接口，删除指定对象。参数意义依次为存储桶名称，对象名称。

AmazonS3 s3 = this.getS3();
//After getting the S3 connection, call the method to delete the object.
s3.deleteObject(bucketName, objectName);

区域的创建
对于区域操作， SequoiaS3 提供的有 Rest 接口，通过构建 CreateRegion 请求实现新建区域，同时还可以对同名区域进行更新配置。

//Operation type
String action = "CreateRegion";
//Build the url and it contains operation type and region name.
String urlStr = "http://127.0.0.1:8002/region/?Action="+action+"&RegionName="+regionName;
post = new HttpPost(urlStr);
//Request the header to set the AccessKeyID and SecreatKeyID of S3
post.setHeader("Authorization","AWS ABCDEFGHIJKLMNOPQRST:abcdefghijklmnopqrstuvwxyz0123456789ABCD");
//The required parameters are in XML format, and they use dom4j tool to build.
Document document = DocumentHelper.createDocument();
//Set the master node
Element root = document.addElement("RegionConfiguration");
//Add the secondary node
Element dataCSShardingType = root.addElement("DataCSShardingType");
Element dataCLShardingType = root.addElement("DataCLShardingType");
//Set the node value
dataCSShardingType.addText("year");
dataCLShardingType.addText("month");
//Convert to the string with XML format
String strXML = document.asXML();
//Put parameters into the body of the Post request
StringEntity postingString = new StringEntity(strXML,"utf-8");
postingString.setContentType("application/xml");
post.setEntity(postingString);

区域的查询
对于区域操作，SequoiaS3 提供的有 Res t接口，通过构建 ListRegions 请求实现获得区域列表。该操作是查询出当前 S3 集群中区域的列表。

//Operation type
String action = "ListRegions";
//Region name
String regionName = "firstregion";
//Build url, and it contains operation type and region name.
String urlStr = "http://127.0.0.1:8002/region/?Action="+action;
post = new HttpPost(urlStr);
//Request the header to set the AccessKeyID and SecreatKeyID of S3
post.setHeader("Authorization"
               ,"AWS ABCDEFGHIJKLMNOPQRST:abcdefghijklmnopqrstuvwxyz0123456789ABCD");

区域的删除
对于区域操作，SequoiaS3 提供的有Rest接口，通过构建 DeleteRegion 请求实现删除指定区域。该操作是查询出当前 S3 集群中区域的列表。

//Operation type
String action = "DeleteRegion";
//Build url, and it contains operation type and region name.
String urlStr = "http://127.0.0.1:8002/region/?Action="+action+"&RegionName="+regionName;
post = new HttpPost(urlStr);
//Request the header to set the AccessKeyID and SecreatKeyID of S3
post.setHeader("Authorization",
               "AWS ABCDEFGHIJKLMNOPQRST:abcdefghijklmnopqrstuvwxyz0123456789ABCD");

设置元数据
在将文件上传为 S3 对象时，可以在上传的同时设定元数据参数，然后再上传。

//Get the S3 connection
AmazonS3 s3 = this.getS3();
//Create a bucket to use
s3.createBucket(bucketName);
//Create file input stream
File file = new File("/opt/sequoiadb/version.conf");
InputStream inputStream = new FileInputStream(file);
//Get metadata object
ObjectMetadata objectMetadata = new ObjectMetadata();
//Set metadata properties
objectMetadata.setContentLength(file.length());
objectMetadata.setContentLanguage("CH");
objectMetadata.setContentEncoding("utf8");
objectMetadata.setContentType("text/plain");
//Save the uploaded file as an object and set the object metadata
s3.putObject(bucketName,objectName,inputStream,objectMetadata);
inputStream.close();

查看元数据
在一个已有的 S3 实例中，可以通过 getObjectMetadata(String str,String str1) 函数获得指定对象的元数据对象。

//Get the S3 connection
AmazonS3 s3 = this.getS3();
//Get metadata object of the specified object
ObjectMetadata objectMetadata =
    s3.getObjectMetadata(bucketName,objectName);
//Get metadata properties
String contentLanguage = objectMetadata.getContentLanguage();
String contentEncoding = objectMetadata.getContentEncoding();
String contentType = objectMetadata.getContentType();
//Print metadata properties
System.out.println("contentLanguage:"+contentLanguage);
System.out.println("contentEncoding:"+contentEncoding);
System.out.println("contentType:"+contentType);
//Clean up the environment
s3.deleteObject(bucketName,objectName);
s3.deleteBucket(bucketName);

实验 4 创建 LOB 对象存储及数据操作
在这里插入图片描述
获得SequoiaDB连接
在使用 Java 开发 SequoiaDB 时，需要先连接到 SequoiaDB ，然后通过该连接对SequoiaDB 集群进行操作。

// addr, port, username, password
sequoiadb = new Sequoiadb("sdbserver1", 11810, "", "");

上传LOB
在 Java 中上传 LOB 时需要先创建一个 LOB 对象，然后通过 LOB 对象向集群中写入数据。

//Get the collection space object
CollectionSpace cs = sequoiadb1.createCollectionSpace(csName);
//Get collection object
DBCollection cl = cs.createCollection(clName);

//Create the Lob
DBLob lob = cl.createLob();
ObjectId id = lob.getID();
//Print oid, and a unique oid will be generated when the Lob object is created
String s = id.toString();
fileInputStream = new FileInputStream(file);
//Write data to the Lob
lob.write(fileInputStream);
//Close the Lob
lob.close();

查询LOB
SequoiaDB 在 DBCollection 类中提供了 listLob() 接口，可以获得指定集合内的 LOB 列表。

//Get the collection space object
CollectionSpace cs = sequoiadb1.getCollectionSpace(csName);
//Get collection object
DBCollection cl = cs.getCollection(clName);
//Get the list of Lob in the collection
DBCursor cursor = cl.listLobs();
try {
//Traverse the Lob list and output Lob information
while (cursor.hasNext()) {
BSONObject record = cursor.getNext();
System.out.println(record.toString());
}
} catch (Exception e) {
e.printStackTrace();
} finally {
cursor.close();
}

下载LOB
SequoiaDB 在 DBCollection 类中提供了 openLob() 接口，可以获得集合内的指定 LOB 的引用，然后读取 LOB，通过文件输出流写入到本地。
同时将上一小节获得的 oid 粘贴到下方代码的第二行的双引号内:

//Get the specified Lob object by oid
DBLob dbLob = cl.openLob(new ObjectId(""));
FileOutputStream fileOutputStream = null;
try {
//Get the file output stream, and set the file path and file name
fileOutputStream = new FileOutputStream(new File("/home/shiyanlou/Desktop/sequoiadb.txt"));
//Read the Lob and write to the local
dbLob.read(fileOutputStream);
} catch (FileNotFoundException e) {
e.printStackTrace();
}finally {
dbLob.close();
}

删除LOB
SequoiaDB 在 DBCollection 类中提供了 removeLob() 接口，传入要删除对象的 oid 就可以删除指定 LOB。
并将上一步用到的 oid 再次粘贴到下方代码的最后一行的双引号内。

//Get the collection space object
CollectionSpace cs = sequoiadb1.getCollectionSpace(csName);
//Get the collection object
DBCollection cl = cs.getCollection(clName);
//Delete the specified Lob
cl.removeLob(new ObjectId(""));

查询LOB
查询功能的实现复用在 LOB 的创建和查询小节中实现的查询 LOB 功能。
在这里插入图片描述
实验 5 LOB 对象存储分区规则
在这里插入图片描述
创建水平分区集合
可以通过 Java SDK 提供的 createCollection(String collectionName, BSONObject options) 接口创建指定属性的集合。

Sequoiadb sequoiadb = this.getSdbConnect();
CollectionSpace cs = sequoiadb.createCollectionSpace(csName);
BasicBSONObject cl_options = new BasicBSONObject();
//set collection params
cl_options.put("ShardingType","hash");
cl_options.put("ShardingKey",new BasicBSONObject("sid",1));
cl_options.put("Partition",4096);
cl_options.put("AutoSplit",true);
//create collection
cs.createCollection(clName,cl_options);

在这里插入图片描述
垂直分区的创建
创建主集合

//Create the main collection, and the main table must be divided with range
Sequoiadb sequoiadb = this.getSdbConnect();
sequoiadb.createCollectionSpace(csName);
BasicBSONObject mainOptions = new BasicBSONObject();
// Use the field date as the main collection partition key, and this field is date type.
mainOptions.put("ShardingKey",new BasicBSONObject("date",1));
mainOptions.put("ShardingType", "range");
//Claim the main collection
mainOptions.put("IsMainCL", true);
mainOptions.put("LobShardingKeyFormat","YYYYMMDD");
DBCollection mainCL = sequoiadb.getCollectionSpace(csName).
    createCollection(mainClName, mainOptions);

创建子集合：

BasicBSONObject subOptions = new BasicBSONObject();
//Set the partition key
subOptions.put("ShardingKey",new BasicBSONObject("date",1));
//Set the partition method
subOptions.put("ShardingType", "hash");
DBCollection subCL1 = sequoiadb.getCollectionSpace(csName).
    createCollection(slaveClName, subOptions);

挂载子集合

Sequoiadb sequoiadb = this.getSdbConnect();
//The parameters when mounting are mainly the upper and lower boundaries of the sub-table settings
BasicBSONObject attachOptions = new BasicBSONObject();
attachOptions.put("LowBound", new BasicBSONObject("date", lowBound));
attachOptions.put("UpBound", new BasicBSONObject("date", upBound));
//Get the main table object
DBCollection mainCL = sequoiadb.getCollectionSpace(mainCsName).
    getCollection(mainClName);
//Get the subtable object
DBCollection subCL = sequoiadb.getCollectionSpace(slaveCsName).
    getCollection(slaveClName);
//Mount the subtable to the main table
mainCL.attachCollection(subCL.getFullName(), attachOptions);

实验 6 SequoiaFS 文件系统操作
在SequoiaFS上写入文件

InputStream put = new FileInputStream("/home/sdbadmin/sequoiadb.txt");
OutputStream out  = new FileOutputStream("/opt/sequoiafs/mountpoint/sequoiadb.txt");
byte[] cbuf = new byte[1024];
int len = 1024;
//How many bytes of file are read at a time
while((len = put.read(cbuf))!= -1){
    out.write(cbuf,0,len);
    out.flush();
}
put.close();

在SequoiaFS上读取文件

//Get the file input stream
InputStreamReader put = new InputStreamReader(new FileInputStream("/opt/sequoiafs/mountpoint/sequoiadb.txt"), "utf-8");
char[] cbuf = new char[1024];
int len = 1024;
//Read the file content and output to console
while((len = put.read(cbuf))!= -1){
    System.out.println(new String(cbuf, 0, len));
}
put.close();

实验 7 LOB 存储设计常见问题
在这里插入图片描述在这里插入图片描述
创建水平分区集合

//set collection params
BasicBSONObject cl_options = new BasicBSONObject();
cl_options.put("ShardingType","hash");
cl_options.put("ShardingKey",new BasicBSONObject("sid",1));
cl_options.put("Partition",4096);
cl_options.put("AutoSplit",true);
//create collection
cs.createCollection(clName,cl_options);

在这里插入图片描述
创建垂直分区集合:
主集合参数设置：

// Use the field date as the main collection partition key, and this field is date type.
mainOptions.put("ShardingKey",new BasicBSONObject("date",1));
mainOptions.put("ShardingType", "range");
//Claim the main collection
mainOptions.put("IsMainCL", true);
// Create main collection
mainOptions.put("LobShardingKeyFormat","YYYYMMDD");

子集合参数设置：

//Set the partition key
subOptions.put("ShardingKey",new BasicBSONObject("date",1));
//Set the partition method
subOptions.put("ShardingType", "hash");

挂载参数设置。

attachOptions.put("LowBound",new BasicBSONObject("date", "20200101"));
attachOptions.put("UpBound",new BasicBSONObject("date", "20200201"));

在这里插入图片描述
生成OID

ObjectId id1 =new ObjectId(new Date(),1234,1234);
ObjectId id2 =new ObjectId(new Date(),1234,4321);
System.out.println("time identical,machine inequality,inc inequality");
System.out.println("id1 :"+id1.toString());
System.out.println("id2 :"+id2.toString());

Date date1 = sdf.parse("20190101");
Date date2 = sdf.parse("20200201");
System.out.println("time inequality,machine identical,inc identical");
ObjectId id3 =new ObjectId(date1,465214,1234);
ObjectId id4 =new ObjectId(date2,465214,4312);
System.out.println("id3 :"+id3.toString());
System.out.println("id4 :"+id4.toString());

System.out.println("time identical,machine inequality,inc inequality");
ObjectId id5 =new ObjectId(new Date(),1234,1234);
ObjectId id6 =new ObjectId(new Date(),4321,1234);
System.out.println("id5 :"+id5.toString());
System.out.println("id6 :"+id6.toString());

在这里插入图片描述
创建指定 LobPageSize 的集合空间

//Set the LobPageSize value
options.put("LobPageSize",65536);
//Create a collection space with the specified LobPageSize
CollectionSpace cs = sequoiadb.createCollectionSpace(csName,options);
cs.createCollection(clName);

实验 8
影像数据存储案例

存储表设计：
在这里插入图片描述
创建 LOB

Sequoiadb sequoiadb1 = this.getSdbConnect();
//Get the collection space object
CollectionSpace cs = sequoiadb1.createCollectionSpace(csName);
//Get the collection object
DBCollection cl = cs.createCollection(clName);
//Create the Lob
DBLob lob = cl.createLob();
//Call a function that inserts to the LOB metadata
this.putLobData(csName,clName,lob,file);
FileInputStream fileInputStream = new FileInputStream(file);
//Write data to the Lob
lob.write(fileInputStream);
//Close the Lob
lob.close();

插入LOB元数据:

Sequoiadb sequoiadb1 = this.getSdbConnect();
//Get the Lobid
ObjectId id = lob.getID();
String lobid = id.toString();
//Get the file name to upload
String fileName = file.getName();
//Get the date of today
Date date = this.getDate(0);
//Get the file size
long file_length = file.length();
//Encapsulate metadata attributes
BSONObject bsonObject = new BasicBSONObject();
bsonObject.put("filename",fileName);
bsonObject.put("lobid",lobid);
bsonObject.put("putdate",date);
bsonObject.put("file_length",file_length);
//Get the collection object
DBCollection collection = sequoiadb.getCollectionSpace(csName).getCollection(clName);
//Insert data into the collection
collection.insert(bsonObject);

查询数据:

Sequoiadb sequoiadb = this.getSdbConnect();
//Get the collection object
DBCollection collection = sequoiadb.getCollectionSpace(csName).getCollection(clName);
BSONObject matcher = new BasicBSONObject();
//Add search condition
matcher.put("filename",fileName);
//Get the date of yesterday
Date date = this.getDate(-1);
//Query files generated after yesterday
matcher.put("putdate",new BasicBSONObject("$gt", date));
//Execute query
DBCursor cursor = collection.query(matcher,null,null,null);
//Output query results
while (cursor.hasNext()) {
    BSONObject record = cursor.getNext();
    System.out.println(record.toString());
}
//Query the LOB in collection
DBCursor lob_cursor = collection.listLobs();
while (lob_cursor.hasNext()) {
    BSONObject record = lob_cursor.getNext();
    System.out.println(record.toString());
}