曲靖做网站需要多少钱,sku电商是什么意思,汝阳网站建设哪家好,服务好的高端网站建设问题及背景#xff1a;同一用户的三家公司的物理集群合并#xff0c;合并后用dataspacekerberos控制不同公司对集群资料的访问权限#xff0c;三家公司分别使用独立的kerberos票据访问#xff0c;特定的namespace,而生产环境部署的kylin-2.0/2.1只能保存cuboid到hbase 的 d…问题及背景同一用户的三家公司的物理集群合并合并后用dataspacekerberos控制不同公司对集群资料的访问权限三家公司分别使用独立的kerberos票据访问特定的namespace,而生产环境部署的kylin-2.0/2.1只能保存cuboid到hbase 的 default namespace。
解决的过程及方案 1、经多方查找发现 Kylin源码github项目最新的master分支已经支持更改存储的hbase的默认namespace但是官方对外发布的最新安装包才更新到Kylin-2.1版本且最新源码并没有经过生产的验证不敢冒然部署于是想办法在现有的版本上重新编译源码、打包。 2、经多方搜索确认更改hbase namespace的commit是kylin-2846于是查找相关实现的细节。
解决思路 1、首先按照github上提供的修改方法更改了https://github.com/apache/kylin/commit/ffcbdb7f4288fa6011f018163bdce9602415cf10 中的五个类重新编译打包执行到convert cuboid Data to HFile这一步的时候抛出异常找不到Hbase表
org.apache.hadoop.hbase.TableNotFoundException:ISW:KYLIN_F3KD3ZC1LU
at org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:575)
而这时我在hbase shell指定的namespace查看已经生成了hbase表为什么会找不到呢继续查找原因 2、继续找Kylin-2846相关的内容发现有人遇到同样的问题地址https://issues.apache.org/jira/browse/KYLIN-2846 原来github上给出的代码少了很关键的一步
---.../main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 -1 file changed, 1 insertion(), 1 deletion(-)diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 9f9b382..7a583ca 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.javab/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java-92,7 92,7 public class CubeHFileJob extends AbstractHadoopJob {attachCubeMetadata(cube, job.getConfiguration());Configuration hbaseConf HBaseConfiguration.create(getConf());
- HTable htable new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase());HTable htable new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME));// Automatic config !HFileOutputFormat.configureIncrementalLoad(job, htable);
--
1.9.1
具体的修改步骤如下From 004e1bcb983442a1a2c69f7d1d6c09dc29d4301e Mon Sep 17 00:00:00 2001
From: Liu Shaohui liushaohuixiaomi.com
Date: Tue, 5 Sep 2017 15:50:29 0800
Subject: [PATCH] Add a config of hbase namespace for cube storage---.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 core-common/src/main/resources/kylin-defaults.properties | 3 core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 2 .../main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java | 2 -.../main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 -5 files changed, 11 insertions(), 2 deletions(-)diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 749b515..3f53023 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.javab/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java-712,6 712,10 abstract public class KylinConfigBase implements Serializable {return StorageURL.valueOf(url);} public String getHBaseStorageNameSpace() {return getOptional(kylin.storage.hbase.namespace, default);}
public String getHBaseClusterFs() {return getOptional(kylin.storage.hbase.cluster-fs, );}
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index a3277ce..34fb5a0 100644
--- a/core-common/src/main/resources/kylin-defaults.propertiesb/core-common/src/main/resources/kylin-defaults.properties-65,6 65,9 kylin.source.hive.redistribute-flat-tabletrue# The storage for final cube file in hbasekylin.storage.urlhbase# The namespace for hbase storage
kylin.storage.hbase.namespacedefault
# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]kylin.storage.hbase.compression-codecnonediff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 043993c..484b977 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.javab/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java-684,11 684,13 public class CubeManager implements IRealizationProvider {}private String generateStorageLocation() {String namespace config.getHBaseStorageNameSpace();String namePrefix IRealizationConstants.CubeHbaseStorageLocationPrefix;String tableName ;Random ran new Random();do {StringBuffer sb new StringBuffer();sb.append(namespace).append(:);sb.append(namePrefix);for (int i 0; i HBASE_TABLE_LENGTH; i) {sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 30616c5..d1caccb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.javab/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java-47,7 47,7 public class BulkLoadJob extends AbstractHadoopJob {options.addOption(OPTION_CUBE_NAME);parseOptions(options, args);- String tableName getOptionValue(OPTION_HTABLE_NAME).toUpperCase();String tableName getOptionValue(OPTION_HTABLE_NAME);// e.g// /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/// end with /
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 1a624c4..9f9b382 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.javab/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java-68,7 68,7 public class CubeHFileJob extends AbstractHadoopJob {Path partitionFilePath new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));Path output new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String cubeName getOptionValue(OPTION_CUBE_NAME).toUpperCase();String cubeName getOptionValue(OPTION_CUBE_NAME);CubeManager cubeMgr CubeManager.getInstance(KylinConfig.getInstanceFromEnv());--
1.9.1
2、、、From 677400ee7b4cf5be0ddc9b595d80e02a0ba87c12 Mon Sep 17 00:00:00 2001
From: Liu Shaohui liushaohuixiaomi.com
Date: Fri, 15 Sep 2017 14:34:18 0800
Subject: [PATCH] Addium for KYLIN-2846---.../main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 -1 file changed, 1 insertion(), 1 deletion(-)diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 9f9b382..7a583ca 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.javab/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java-92,7 92,7 public class CubeHFileJob extends AbstractHadoopJob {attachCubeMetadata(cube, job.getConfiguration());Configuration hbaseConf HBaseConfiguration.create(getConf());
- HTable htable new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase());HTable htable new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME));// Automatic config !HFileOutputFormat.configureIncrementalLoad(job, htable);
--
1.9.1
3、clean htablediff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 3728ea1..2d95542 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.javab/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java-46,15 46,28 public class StorageCleanJobHbaseUtil {public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {Configuration conf HBaseConfiguration.create();
- CubeManager cubeMgr CubeManager.getInstance(KylinConfig.getInstanceFromEnv());KylinConfig config KylinConfig.getInstanceFromEnv();CubeManager cubeMgr CubeManager.getInstance(config);// get all kylin hbase tablestry (HBaseAdmin hbaseAdmin new HBaseAdmin(conf)) {
- String tableNamePrefix IRealizationConstants.SharedHbaseStorageLocationPrefix;String namespace config.getHBaseStorageNameSpace();StringBuffer sb new StringBuffer();String tableNamePrefix null;if(namespace.equals(default) || namespace.equals()){tableNamePrefix IRealizationConstants.SharedHbaseStorageLocationPrefix;}else{sb.append(config.getHBaseStorageNameSpace()).append(:);sb.append(IRealizationConstants.SharedHbaseStorageLocationPrefix);tableNamePrefix sb.toString();}
HTableDescriptor[] tableDescriptors hbaseAdmin.listTables(tableNamePrefix .*);
ListString allTablesNeedToBeDropped new ArrayListString();for (HTableDescriptor desc : tableDescriptors) {
String host desc.getValue(IRealizationConstants.HTableTag);
- if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {if (config.getMetadataUrlPrefix().equalsIgnoreCase(host)) {//only take care htables that belongs to self, and created more than 2 daysallTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());}
修改完后core-common, core-cure, storage-hbase三个目录重新打jar包覆盖原Kylin安装目录KYLIN_HOME/tomcat/webapps/kylin.war下对应的jar包重启Kylin问题解决。
