From e9d828f8051263d79e20686ddcb14e51b5952986 Mon Sep 17 00:00:00 2001 From: HongW2019 Date: Fri, 3 Sep 2021 12:58:52 +0800 Subject: [PATCH] [SQL-DS-CACHE-201] Update guide for OAP 1.2.0 --- docs/Developer-Guide.md | 2 +- docs/User-Guide.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 7294a930a..93fac8a8f 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -36,7 +36,7 @@ To use optimized Plasma cache with OAP, you need following components: ``` cd /tmp git clone https://github.com/oap-project/arrow.git -cd arrow && git checkout arrow-4.0.0-oap-1.1.1 +cd arrow && git checkout v4.0.0-oap-1.2.0 cd cpp mkdir release cd release diff --git a/docs/User-Guide.md b/docs/User-Guide.md index 8b19464f7..80cbb20d2 100644 --- a/docs/User-Guide.md +++ b/docs/User-Guide.md @@ -263,7 +263,7 @@ Socket Configuration -> Intel UPI General Configuration -> Stale AtoS : Disable For more information you can refer to [Quick Start Guide: Provision Intel® Optane™ DC Persistent Memory](https://software.intel.com/content/www/us/en/develop/articles/quick-start-guide-configure-intel-optane-dc-persistent-memory-on-linux.html) -- SQL Data Source Cache uses Plasma as a node-level external cache service, the benefit of using external cache is data could be shared across process boundaries. [Plasma](http://arrow.apache.org/blog/2017/08/08/plasma-in-memory-object-store/) is a high-performance shared-memory object store and a component of [Apache Arrow](https://github.com/apache/arrow). We have modified Plasma to support PMem, and make it open source on [oap-project-Arrow](https://github.com/oap-project/arrow/tree/arrow-4.0.0-oap-1.1.1) repo. If you have finished [OAP Installation Guide](OAP-Installation-Guide.md), Plasma will be automatically installed and then you just need copy `arrow-plasma-4.0.0.jar` to `$SPARK_HOME/jars`. For manual building and installation steps you can refer to [Plasma installation](./Developer-Guide.md#Plasma-installation). +- SQL Data Source Cache uses Plasma as a node-level external cache service, the benefit of using external cache is data could be shared across process boundaries. [Plasma](http://arrow.apache.org/blog/2017/08/08/plasma-in-memory-object-store/) is a high-performance shared-memory object store and a component of [Apache Arrow](https://github.com/apache/arrow). We have modified Plasma to support PMem, and make it open source on [oap-project-Arrow](https://github.com/oap-project/arrow/tree/arrow-4.0.0-oap-1.2) repo. If you have finished [OAP Installation Guide](OAP-Installation-Guide.md), Plasma will be automatically installed and then you just need copy `arrow-plasma-4.0.0.jar` to `$SPARK_HOME/jars`. For manual building and installation steps you can refer to [Plasma installation](./Developer-Guide.md#Plasma-installation). - Refer to configuration below to apply external cache strategy and start Plasma service on each node and start your workload. @@ -280,11 +280,11 @@ spark.executor.instances 6 spark.sql.extensions org.apache.spark.sql.OapExtensions # absolute path of the jar on your working node, when in Yarn client mode -spark.files $HOME/miniconda2/envs/oapenv/oap_jars/plasma-sql-ds-cache--with-spark-.jar,$HOME/miniconda2/envs/oapenv/oap_jars/pmem-common--with-spark-.jar +spark.files $HOME/miniconda2/envs/oapenv/oap_jars/plasma-sql-ds-cache--with-spark-.jar,$HOME/miniconda2/envs/oapenv/oap_jars/pmem-common--with-spark-.jar,$HOME/miniconda2/envs/oapenv/oap_jars/arrow-plasma-4.0.0.jar # relative path to spark.files, just specify jar name in current dir, when in Yarn client mode -spark.executor.extraClassPath ./plasma-sql-ds-cache--with-spark-.jar:./pmem-common--with-spark-.jar +spark.executor.extraClassPath ./plasma-sql-ds-cache--with-spark-.jar:./pmem-common--with-spark-.jar:./arrow-plasma-4.0.0.jar # absolute path of the jar on your working node,when in Yarn client mode -spark.driver.extraClassPath $HOME/miniconda2/envs/oapenv/oap_jars/plasma-sql-ds-cache--with-spark-.jar:$HOME/miniconda2/envs/oapenv/oap_jars/pmem-common--with-spark-.jar +spark.driver.extraClassPath $HOME/miniconda2/envs/oapenv/oap_jars/plasma-sql-ds-cache--with-spark-.jar:$HOME/miniconda2/envs/oapenv/oap_jars/pmem-common--with-spark-.jar:$HOME/miniconda2/envs/oapenv/oap_jars/arrow-plasma-4.0.0.jar # for parquet file format, enable binary cache spark.sql.oap.parquet.binary.cache.enabled true