Skip to content

Commit be081dd

Browse files
JimvinJimvin
authored
Improvements to quickstart (#11)
* Moved service CRDs into separate files. Bug fixes and improvements. * Added support for Trino and Hive * Moved data to /data rather than /tmp. Updated URLs for Hive CRDs. * Added OPA * Bundled CRDs with the quickstart script. Changed references to node labels from arch to os. * Added regorule operator. * Minor updates to README.md
1 parent e2c4dd7 commit be081dd

36 files changed

+3957
-202
lines changed

‎quickstart/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,17 @@
22

33
## Quickstart Script
44
stackable-quickstart.sh is a shell script that will install Stackable on a single node, ideal for development, testing and evaulation. It will get your from a bare OS installation to a working cluster in a few minutes.
5+
6+
7+
## Requirements
8+
Supported operating systems are:
9+
- Ubuntu 20.04
10+
- Debian 10
11+
- CentOS 7 and 8
12+
- RHEL 7 and 8
13+
14+
- The host running Stackable Quickstart needs Internet access in order to download
15+
- Resource requirement (RAM, CPU) will depend on the number and type of services you are deploying using quickstart
16+
17+
## Configuration
18+
In most cases you can use the default configuration to create a Quickstart cluster. You can customise the service configuration by updating the files in the conf/ directory.

‎quickstart/conf/hive.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
---
2+
apiVersion: hive.stackable.tech/v1alpha1
3+
kind: HiveCluster
4+
metadata:
5+
name: simple-derby
6+
spec:
7+
version: 2.3.9
8+
metastore:
9+
roleGroups:
10+
default:
11+
selector:
12+
matchLabels:
13+
kubernetes.io/os: stackable-linux
14+
replicas: 1
15+
config:
16+
javaHome: /usr/lib/jvm/java-11-openjdk-amd64/
17+
metricsPort: 11111
18+
database:
19+
connString: jdbc:derby:;databaseName=/data/hive/metastore_db;create=true
20+
user: APP
21+
password: mine
22+
dbType: derby
23+
warehouseDir: /data/hive/warehouse
24+
25+
s3Connection:
26+
endPoint: changeme
27+
accessKey: changeme
28+
secretKey: changeme
29+
sslEnabled: false
30+
pathStyleAccess: true

‎quickstart/conf/kafka.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
apiVersion: kafka.stackable.tech/v1alpha1
3+
kind: KafkaCluster
4+
metadata:
5+
name: simple
6+
spec:
7+
version:
8+
kafka_version: 2.8.0
9+
zookeeperReference:
10+
namespace: default
11+
name: simple
12+
brokers:
13+
roleGroups:
14+
default:
15+
selector:
16+
matchLabels:
17+
kubernetes.io/os: stackable-linux
18+
replicas: 1
19+
config:
20+
logDirs: "/data/kafka/kafka-logs"
21+
metricsPort: 9606

‎quickstart/conf/nifi.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
---
2+
apiVersion: nifi.stackable.tech/v1alpha1
3+
kind: NifiCluster
4+
metadata:
5+
name: simple
6+
spec:
7+
metricsPort: 8428
8+
version: "1.13.2"
9+
zookeeperReference:
10+
name: simple
11+
namespace: default
12+
chroot: /nifi
13+
nodes:
14+
roleGroups:
15+
default:
16+
selector:
17+
matchLabels:
18+
kubernetes.io/os: stackable-linux
19+
replicas: 1
20+
config:
21+
httpPort: 10000
22+
protocolPort: 10443
23+
loadBalancePort: 6342

‎quickstart/conf/opa.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
apiVersion: opa.stackable.tech/v1alpha1
3+
kind: OpenPolicyAgent
4+
metadata:
5+
name: simple
6+
spec:
7+
version: "0.27.1"
8+
servers:
9+
roleGroups:
10+
default:
11+
selector:
12+
matchLabels:
13+
kubernetes.io/os: stackable-linux
14+
config:
15+
port: 8181
16+
repoRuleReference: "http://localhost:3030/opa/v1"

‎quickstart/conf/regorule.yaml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
---
2+
apiVersion: opa.stackable.tech/v1alpha1
3+
kind: RegoRule
4+
metadata:
5+
name: trino2
6+
spec:
7+
rego: |
8+
package trino
9+
10+
users = {"admin":{"schemas":{"nyc_taxi_trips":{"read":true,"write":true}},"tables":{"yellow_taxi_trips":{"read":true,"write":null},"fhv_trips":{"read":true,"write":true}}}}
11+
12+
default can_access_table = false
13+
can_access_table {
14+
user_can_read_table
15+
}
16+
17+
default can_create_table = false
18+
can_create_table {
19+
user_can_write_table
20+
}
21+
22+
default can_drop_table = false
23+
can_drop_table {
24+
user_can_write_table
25+
}
26+
27+
default can_show_tables = false
28+
can_show_tables {
29+
user_can_read_table
30+
}
31+
32+
default can_access_schema = false
33+
can_access_schema {
34+
user_can_read_schema
35+
}
36+
37+
default can_create_schema = false
38+
can_create_schema {
39+
user_can_write_schema
40+
}
41+
42+
default can_drop_schema = false
43+
can_drop_schema {
44+
user_can_write_schema
45+
}
46+
47+
default can_show_schemas = false
48+
can_show_schemas {
49+
user_can_read_schema
50+
}
51+
52+
default can_access_catalog = false
53+
can_access_catalog {
54+
is_valid_user
55+
}
56+
57+
default can_execute_query = false
58+
can_execute_query {
59+
is_valid_user
60+
}
61+
62+
default can_select_from_columns = false
63+
can_select_from_columns {
64+
is_valid_user
65+
}
66+
67+
default can_view_query_owned_by = false
68+
can_view_query_owned_by {
69+
is_valid_user
70+
}
71+
72+
user_can_read_table {
73+
users[input.input.user.name].tables[input.input.request.table.table].read == true
74+
}
75+
76+
user_can_write_table {
77+
users[input.input.user.name].tables[input.input.request.table.table].write == true
78+
}
79+
80+
user_can_read_schema {
81+
users[input.input.user.name].schemas[input.input.request.table.schema].read == true
82+
}
83+
84+
user_can_write_schema {
85+
users[input.input.user.name].schemas[input.input.request.table.schema].write == true
86+
}
87+
88+
is_valid_user {
89+
_ = users[input.input.user.name]
90+
}

‎quickstart/conf/spark.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
---
2+
apiVersion: spark.stackable.tech/v1alpha1
3+
kind: SparkCluster
4+
metadata:
5+
name: simple
6+
spec:
7+
version: "3.0.1"
8+
config:
9+
logDir: "file:///data/spark/logs"
10+
enableMonitoring: true
11+
masters:
12+
roleGroups:
13+
default:
14+
selector:
15+
matchLabels:
16+
kubernetes.io/os: stackable-linux
17+
replicas: 1
18+
config:
19+
masterPort: 7078
20+
masterWebUiPort: 8081
21+
workers:
22+
roleGroups:
23+
2core2g:
24+
selector:
25+
matchLabels:
26+
kubernetes.io/os: stackable-linux
27+
replicas: 1
28+
config:
29+
cores: 2
30+
memory: "2g"
31+
workerPort: 3031
32+
workerWebUiPort: 8083
33+
historyServers:
34+
roleGroups:
35+
default:
36+
selector:
37+
matchLabels:
38+
kubernetes.io/os: stackable-linux
39+
replicas: 1
40+
config:
41+
historyWebUiPort: 18081

‎quickstart/conf/stackable-repo.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
apiVersion: "stable.stackable.de/v1"
3+
kind: Repository
4+
metadata:
5+
name: stackablepublic
6+
spec:
7+
repo_type: StackableRepo
8+
properties:
9+
url: https://repo.stackable.tech/repository/packages/

‎quickstart/conf/trino.yaml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
---
2+
apiVersion: trino.stackable.tech/v1alpha1
3+
kind: TrinoCluster
4+
metadata:
5+
name: simple
6+
spec:
7+
version: "0.0.362"
8+
nodeEnvironment: production
9+
hiveReference:
10+
namespace: default
11+
name: simple-derby
12+
opa:
13+
namespace: default
14+
name: simple
15+
authorization:
16+
package: trino
17+
permissions:
18+
admin:
19+
schemas:
20+
read: true
21+
write: true
22+
tables:
23+
yellow_taxi_trips:
24+
read: true
25+
write: true
26+
fhv_trips:
27+
read: true
28+
write: true
29+
bob:
30+
schemas:
31+
read: false
32+
write: false
33+
tables:
34+
yellow_taxi_trips:
35+
read: true
36+
s3Connection:
37+
endPoint: changeme
38+
accessKey: changeme
39+
secretKey: changeme
40+
sslEnabled: false
41+
pathStyleAccess: true
42+
coordinators:
43+
roleGroups:
44+
default:
45+
selector:
46+
matchLabels:
47+
kubernetes.io/os: stackable-linux
48+
config:
49+
coordinator: true
50+
nodeDataDir: /data/trino/coordinator
51+
metricsPort: 10200
52+
httpServerHttpPort: 8080
53+
httpServerHttpsPort: 8443
54+
javaHome: /usr/lib/jvm/java-11-openjdk-amd64/
55+
passwordFileContent: |
56+
alice:$2y$10$HcCa4k9v2DRrD/g7e5vEz.Bk.1xg00YTEHOZjPX7oK3KqMSt2xT8W
57+
bob:$2y$10$xVRXtYZnYuQu66SmruijPO8WHFM/UK5QPHTr.Nzf4JMcZSqt3W.2.
58+
admin:$2y$10$89xReovvDLacVzRGpjOyAOONnayOgDAyIS2nW9bs5DJT98q17Dy5i
59+
serverCertificate: |
60+
-----BEGIN PRIVATE KEY-----
61+
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC9dxebNWP+9qXS
62+
4+pvItGG/fuUGttoujDueCZbo7LHQCBjR9ZBCmoUzn5rgUC2Qgtz+PCJit5pwhzw
63+
TSrvDPoITqgmRsJFbL1nNbbjyRyGLIj0+MQ+ycBQS7sd3zVmMxrjFviVdPm0E/tw
64+
u9lSWhyh6tHB/w65zzuxkbVhWjY0gzAV+NzqaxL2B5sJytO3Y2c/Ffd3oXGs24yP
65+
JL1yfLiJMQhOHZiWIZ3C/26yNoJtDa9rhoXcuVz0QrZdCvEPqxQPR419XMpY71Fj
66+
p8h18sK1Sm7HZelB8PPek4of4hl0s0CwqEP2yR6jT+Mt6TETN+5hnx7LaKujgQ3G
67+
25hF5CSxAgMBAAECggEBALiP54YBN3yqBhTugnNqhpMqnMTEeNGG9p2Gm08Potro
68+
A+TSmMrY+0dzc8EDsucYZRG/AYnqpgh92DTZ5yXhDTvEa5aFvT9l0wdUUumIAmpN
69+
vchtALZtUBKoAwnxYpJA3EM1IEhvH8Ok534E74mCUHwGQ/jnng+w6ORsHTVNbJov
70+
ODe7i1iYufvmm+ogeqFElBHe4TP/fAqUohjdxhCBcrMeS1yML6v3sLARLvBylgrA
71+
74DbojF5w9/449j7qrUduLac4X7xBi//giSqaah0Xf1tRd+MFX9d/Kgi6RK9L1Fq
72+
K4JiG8S1GmHXTR5q9NCFdUfo4jZlH1nT+cfEIKCsK0ECgYEA3JCCe+sDr4TnCKXX
73+
C6E+lI2kAhyz6PQt5iar3Txf2Sugjk+GAtPcGV29S75mz3N7zKxl14F/H+LQkARr
74+
ahYFN6Bjwx5vj4PjrXrXeYdZvfExSnGVWlojlK4lW48CbYRRu/KAJ6NJaYMH/T/M
75+
vtY80UgO04aY6KvLrBAOhpc8eCkCgYEA2+eD1P5H4T1rn379aA6TgjclQGXqmkXS
76+
gLyLqau3A7/mk5c5onxibSaYXeJfuA3cQOKipMzYjvf6xRnhl7WqUBRCHc2D7Xzr
77+
dKvfQI2GQyI/P1XDQcrGtALsTnBBckHdjAXN4LwfkHaWqn/lohVKCd3Zw5k/fPlS
78+
vwfrQB9W+UkCgYEAmYmLG6H0ldkpTq7LLNBZFOYodkPuxUo9LUpcNwyRGEczY/1d
79+
p1g9jfbgD/sB7IVSd57G4ibGDLdCQg6hi1660Qc4ixoDxN6lvhyWJIYg5YUhr9Wz
80+
hwraW+SCIuNBaGVE1FWx+mXDc8x/4j3oVCwkd/b2Qy8eAFGe3S7sParHsEkCgYBi
81+
2BOn34xLsCH6FUiuRy8CrurtoWHsakoOV7GnWGnxI4bu/GCJqsNLMLf4K1K/FmvU
82+
NorI7NLCoLB1fAas/JlLWW/GmaQdUbM8RC0bhQG+l1w6icA0iIL8UvwI3ehtBTIi
83+
SFGcHLNLaiKbDNYZHtyixreldfLVp/QLrfGOZwffEQKBgHPFWPknfmUXFaVnZ8aW
84+
T83G9VE5u8Jcix6qZ24Ulc14DuIWGuSqxrecbGbFApqL6hkpOukE1Dgn5JPI6X7D
85+
vp3h9oZdn15u1w/VoVUIWnkXTIP+TjcCL8EWfSKuMw2Hn7y9MBAob/KGzXK4g+er
86+
wYZWfSn0WEwtYdd7Yto+c9CN
87+
-----END PRIVATE KEY-----
88+
-----BEGIN CERTIFICATE-----
89+
MIIDfTCCAmWgAwIBAgIQVgK/lSAEUE3b9kAL+jX12jANBgkqhkiG9w0BAQsFADAW
90+
MRQwEgYDVQQDDAtFYXN5LVJTQSBDQTAeFw0yMTEwMTQwNzU5MzNaFw0yNDAxMTcw
91+
NzU5MzNaMBwxGjAYBgNVBAMMEXRyaW5vLWNvb3JkaW5hdG9yMIIBIjANBgkqhkiG
92+
9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvXcXmzVj/val0uPqbyLRhv37lBrbaLow7ngm
93+
W6Oyx0AgY0fWQQpqFM5+a4FAtkILc/jwiYreacIc8E0q7wz6CE6oJkbCRWy9ZzW2
94+
48kchiyI9PjEPsnAUEu7Hd81ZjMa4xb4lXT5tBP7cLvZUlocoerRwf8Ouc87sZG1
95+
YVo2NIMwFfjc6msS9gebCcrTt2NnPxX3d6FxrNuMjyS9cny4iTEITh2YliGdwv9u
96+
sjaCbQ2va4aF3Llc9EK2XQrxD6sUD0eNfVzKWO9RY6fIdfLCtUpux2XpQfDz3pOK
97+
H+IZdLNAsKhD9skeo0/jLekxEzfuYZ8ey2iro4ENxtuYReQksQIDAQABo4HAMIG9
98+
MAkGA1UdEwQCMAAwHQYDVR0OBBYEFHq9PJixGbP4v89qUD4Hhjp7ebuGMFEGA1Ud
99+
IwRKMEiAFGxaNmvmzXGNBdGRWdzOcrAiTnfroRqkGDAWMRQwEgYDVQQDDAtFYXN5
100+
LVJTQSBDQYIUKU/Snhl2qjvKYYSi2F0jlPZ5vpcwEwYDVR0lBAwwCgYIKwYBBQUH
101+
AwEwCwYDVR0PBAQDAgWgMBwGA1UdEQQVMBOCEXRyaW5vLWNvb3JkaW5hdG9yMA0G
102+
CSqGSIb3DQEBCwUAA4IBAQBkjv3AnYYuDuHtRmOWsdWwzvDAZ2L4MWjOihN6nr8o
103+
yeHfjAmZ6nYS1x/0R5R6HlHDMpZMT4s4nHKACLQzO8YFJ2BZWyF5TuYllrSf+mbF
104+
iT+HOAcz2v0t/CQqp0AHhIwdk1ANdSOR8nFxuwyOO+tpnNshx1ikURsiIfdeCeEG
105+
bAREGlhwrKodltECsTLt7VOP0YQj7n2cmsStSBkri32z+ya2DD8UuNplOvCxe4Ed
106+
AEsql+pexNMDSsFB3reQuGuyP7+kvP4bU0rMrVmQ4YWQrXeIT06HfZGwk1kE3PND
107+
1IAwXwBkd1196IfsQkm3Qsi+U1E0/vYjfwCtrx1Anx5W
108+
-----END CERTIFICATE-----
109+
workers:
110+
roleGroups:
111+
default:
112+
selector:
113+
matchLabels:
114+
kubernetes.io/os: stackable-linux
115+
replicas: 1
116+
config:
117+
nodeDataDir: /data/trino/worker
118+
metricsPort: 10201
119+
httpServerHttpPort: 8087
120+
javaHome: /usr/lib/jvm/java-11-openjdk-amd64/

‎quickstart/conf/zookeeper.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
apiVersion: zookeeper.stackable.tech/v1alpha1
3+
kind: ZookeeperCluster
4+
metadata:
5+
name: simple
6+
spec:
7+
version: 3.5.8
8+
servers:
9+
roleGroups:
10+
default:
11+
selector:
12+
matchLabels:
13+
kubernetes.io/os: stackable-linux
14+
replicas: 1
15+
config:
16+
adminPort: 12000
17+
metricsPort: 9505
18+
dataDir: /data/zookeeper

0 commit comments

Comments
 (0)