add provision support to cli+aws

configs/cli.py

@@ -18,7 +18,7 @@ def cli():
 @click.argument('input', type=click.File('rb'))
 @click.argument('format')
 @click.argument('output', type=click.File('wb'))
-@click.option('-v', '--vault', 'vault', default='sops', required=False, multiple=True)
+@click.option('-v', '--vault', 'vault', default=['sops'], required=False, multiple=True)
 def transform(input, format, output, vault):
     """Transform INPUT into FORMAT format and output to OUTPUT
     """
@@ -40,15 +40,17 @@ def transform(input, format, output, vault):
 
     logger.info('Initializing transform')
     transform_config = cfg.get_transform_config(format)
-    transform = Transforms[format](transform_config, vault_stack)
+    transform = Transforms[format](transform_config)
 
     logger.info('Transforming')
-    result = transform.transform(cfg)
+    result = transform.transform(cfg, vault_stack)
     print(result)
 
 @cli.command()
 @click.argument('input', type=click.File('rb'))
-def provision(input):
+@click.argument('source_vault')
+@click.argument('target_vault')
+def provision(input, source_vault, target_vault):
     """Read INPUT and store in the vault service
     """
     logger = logging.getLogger("configs")
@@ -57,10 +59,18 @@ def provision(input):
     cfg = Config()
     cfg.read(input)
 
-    logger.info('Fetching vault config')
-    vault_config = cfg.get_vault_config("aws")
+    logger.info('Initializing source vault')
+    source_vault_config = cfg.get_vault_config(source_vault)
+    source_vault = Vaults[source_vault](source_vault_config)
+
+    logger.info('Initializing target valut')
+    target_vault_config = cfg.get_vault_config(target_vault)
+    target_vault = Vaults[target_vault](target_vault_config)
+
+    logger.info('Fetching config data')
+    secrets = cfg.get_merged(source_vault)
+
+    logger.info('Provisioning in target vault')
+    target_vault.provision(secrets)
 
-    vault = VaultAws(vault_config)
 
-    logger.info('Storing')
-    vault.provision(cfg.get_merged())

configs/config.py

@@ -1,11 +1,11 @@
 import yaml
+from copy import deepcopy
 
 try:
     from yaml import CLoader as YamlLoader, CDumper as YamlDumper
 except ImportError:
     from yaml import Loader as YamlLoader, Dumper as YamlDumper
 
-
 class Config:
 
     def __init__(self):
@@ -22,8 +22,21 @@ class Config:
     def export(self):
         return yaml.dump(self.data, default_flow_style=False)
 
-    def get_secrets(self):
-        return self.data['secrets_encrypted']
+    def get_secrets(self, decrypt=None):
+        secrets = self.data['secrets_encrypted']
+        if decrypt is not None:
+            secrets = deepcopy(secrets)
+            self._decrypt(secrets, decrypt)
+        return secrets
+
+    def _decrypt(self, obj, vault, path=[]):
+        for k in obj.keys():
+            path.append(k)
+            if type(obj[k]) is dict:
+                self._decrypt(obj[k], vault, path)
+            else:
+                obj[k] = vault.resolve(self, ".".join(path))
+            path.pop()
 
     def get_meta(self):
         return self.data['meta']
@@ -40,8 +53,8 @@ class Config:
 
         return self.data['vault'][vault]
 
-    def get_merged(self):
-        return Config._merge_dicts(self.data['config'], self.data['secrets_encrypted'])
+    def get_merged(self, decrypt=None):
+        return Config._merge_dicts(self.data['config'], self.get_secrets(decrypt))
 
     def _merge_dicts(a, b, path=None):
         # https://stackoverflow.com/questions/7204805/dictionaries-of-dictionaries-merge

configs/transform/env.py

@@ -3,14 +3,13 @@ import os
 
 class Env:
 
-    def __init__(self, transform_config, vault):
+    def __init__(self, transform_config):
         self.config = transform_config
-        self.vault = vault
 
-    def transform(self, config):
+    def transform(self, config, vault):
         out = []
         for k in self.config["fields"].keys():
-            value = self.vault.resolve(config, self.config["fields"][k])
+            value = vault.resolve(config, self.config["fields"][k])
             if value is None:
                 out.append('# ' + k + '=null')
             else:

configs/vault/aws.py

@@ -15,9 +15,41 @@ class Aws:
         )
         self.secretsmanager = client
 
-    def provision(self, config):
-        secrets = Aws._build_secrets(config.get_secrets())
-        pprint(secrets)
+    def provision(self, secrets):
+        secrets = Aws._build_secrets(secrets)
+
+        for k in secrets.keys():
+            secret_path = k
+            if 'base_path' in self.config:
+                secret_path = self.config['base_path'] + k
+
+            secret_arn = None
+
+            try:
+                get_secret_value_response = self.secretsmanager.get_secret_value(
+                        SecretId=secret_path
+                    )
+                secret_arn = get_secret_value_response['ARN']
+            except ClientError as e:
+                if e.response['Error']['Code'] == 'ResourceNotFoundException':
+                    secret_arn = None
+                else:
+                    raise e
+
+            if secret_arn is None:
+                self.logger.debug('Creating ' + secret_path)
+                self.secretsmanager.create_secret(
+                    Name=secret_path,
+                    # KmsKeyId
+                    SecretString = json.dumps(secrets[k])
+                )
+            else:
+                self.logger.debug('Updating ' + secret_path)
+                self.secretsmanager.put_secret_value(
+                    SecretId=secret_arn,
+                    SecretString=json.dumps(secrets[k])
+                )
+
 
     def _build_secrets(data, path=[]):
         secrets = {}