ping, curl
root@kube01:/etc:]# ping vminio01.encore.encore
PING vminio01.encore.encore (172.17.172.15) 56(84) bytes of data.
64 bytes from 172.17.172.15 (172.17.172.15): icmp_seq=1 ttl=64 time=0.283 ms
64 bytes from 172.17.172.15 (172.17.172.15): icmp_seq=2 ttl=64 time=0.159 ms
64 bytes from 172.17.172.15 (172.17.172.15): icmp_seq=3 ttl=64 time=0.157 ms
^C
--- vminio01.encore.encore ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2000ms
rtt min/avg/max/mdev = 0.157/0.199/0.283/0.061 ms
#!/usr/bin/env bash
bucket=test
file=orders.json
host=vminio01.encore.encore:9000
s3_key='encore'
s3_secret='${REDACTED}'
resource="/${bucket}/${file}"
content_type="application/octet-stream"
date=`date -R`
_signature="GET\n\n${content_type}\n${date}\n${resource}"
signature=`echo -en ${_signature} | openssl sha1 -hmac ${s3_secret} -binary | base64`
curl -v -X GET \
-H "Host: $host" \
-H "Date: ${date}" \
-H "Content-Type: ${content_type}" \
-H "Authorization: AWS ${s3_key}:${signature}" \
http://$host${resource}
(결과 확인)
{"id":1,"amount":1000}
{"id":2,"amount":2000}
* Connection #0 to host vminio01.encore.encore left intact
{"id":3,"amount":5400}
- k8s-minio-ext.yml
$ cat k8s-minio-ext.yml
kind: Service
apiVersion: v1
metadata:
name: vminio01
spec:
type: ExternalName
externalName: vminio01.encore.encore
$ k apply -f k8s-minio-ext.yml
ping, curl
root@debug:/kubectl run -i --tty --rm debug --image=vharbor.encore.encore/library/alicek106/ubuntu:curl --restart=Never bash
root@debug:/# ping vminio01.encore.encore
PING vminio01.encore.encore (172.17.172.15) 56(84) bytes of data.
64 bytes from 172.17.172.15: icmp_seq=1 ttl=63 time=1.83 ms
64 bytes from 172.17.172.15: icmp_seq=2 ttl=63 time=0.426 ms
root@debug:/kubectl run -i --tty --rm debug --image=vharbor.encore.encore/library/alicek106/ubuntu:curl --restart=Never bash
#!/usr/bin/env bash
bucket=test
file=orders.json
host=vminio01.encore.encore:9000
s3_key='encore'
s3_secret='${REDACTED}'
resource="/${bucket}/${file}"
content_type="application/octet-stream"
date=`date -R`
_signature="GET\n\n${content_type}\n${date}\n${resource}"
signature=`echo -en ${_signature} | openssl sha1 -hmac ${s3_secret} -binary | base64`
curl -v -X GET \
-H "Host: $host" \
-H "Date: ${date}" \
-H "Content-Type: ${content_type}" \
-H "Authorization: AWS ${s3_key}:${signature}" \
http://$host${resource}
(결과 확인)
{"id":1,"amount":1000}
{"id":2,"amount":2000}
* Connection #0 to host vminio01.encore.encore left intact
{"id":3,"amount":5400}
vminio01.encore.encore:9000
encore / dpszhdkdpszhdk
main.
$ tree
├── Dockerfile
├── main.py
└── requirements.txt
$ vi main.py
from pyspark import SparkContext
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
def load_config(spark_context: SparkContext):
spark_context._jsc.hadoopConfiguration().set('fs.s3a.access.key', 'encore')
spark_context._jsc.hadoopConfiguration().set('fs.s3a.encoreret.key', 'dpszhdkdpszhdk')
spark_context._jsc.hadoopConfiguration().set('fs.s3a.path.style.access', 'true')
spark_context._jsc.hadoopConfiguration().set('fs.s3a.impl', 'org.apache.hadoop.fs.s3a.S3AFileSystem')
spark_context._jsc.hadoopConfiguration().set('fs.s3a.endpoint', 'vminio01.encore.encore:9000')
spark_context._jsc.hadoopConfiguration().set('fs.s3a.connection.ssl.enabled', 'false')
load_config(spark.sparkContext)
dataframe = spark.read.json('s3a://test/*')
average = dataframe.agg({'amount': 'avg'})
$ vi Dockerfile
FROM vharbor.encore.encore/library/encore-spark-py:1.0.0
USER root
WORKDIR /app
COPY requirements.txt .
RUN pip3 install -r requirements.txt
COPY main.py .
이미지 푸시
$ docker build -t vharbor.encore.encore/test-kube/test-spark-k8s-vminio01:v1.0.0 .
$ docker push vharbor.encore.encore/test-kube/test-spark-k8s-vminio01:v1.0.0
$ vi k8s-minio-ext.yml
kind: Service
apiVersion: v1
metadata:
name: vminio01
spec:
type: ExternalName
externalName: vminio01.encore.encore
$ k apply -f 3_k8s_minio_ext_pod.yml
$ k get pods
NAME READY STATUS RESTARTS AGE
sparkjobextbminio01-driver 0/1 Completed 0 3h13m
sparkoperator-spark-operator-58d4d9b449-fwrwp 1/1 Running 0 8d
$ k logs sparkjobextbminio01-driver
svc.default.svc:7079 in memory (size: 8.2 KiB, free: 116.9 MiB)
21/09/23 01:48:22 INFO BlockManagerInfo: Removed broadcast_3_piece0 on 10.233.100.11:33065 in memory (size: 8.2 KiB, free: 116.9 MiB)
+-----------+
|avg(amount)|
+-----------+
| 2800.0|
+-----------+