Add Nested Shared Folder (NSF) support to CyberArk PAM import

docs/cyberark-pam-import.md

@@ -62,8 +62,13 @@ Keeper Vault/
 | PaloAltoNetworks, CiscoIOS, CiscoASA, JuniperJunos, F5BigIP, CheckPointGAIA | pamMachine | ssh | 22 |
 | CyberArk (internal) | pamMachine | ssh | 22 |
 | BusinessWebsite | login | — | — |
-| (empty platformId) | pamMachine | ssh | 22 |
-| (unknown platformId) | pamMachine | ssh | 22 |
+| Custom/renamed platform resolved by CyberArk metadata or keyword matching | inferred PAM type | inferred | inferred |
+| Empty/unknown platform with no metadata or keyword match | login | — | — |
+
+For an unresolved platform, fields not already represented by the login's title,
+login, password, URL, or CyberArk notes are preserved as text custom fields.
+Nested account metadata is flattened using dotted field names. Raw secret-bearing
+payload keys are excluded from custom fields.
 
 ### Field Mapping
 
@@ -167,6 +172,9 @@ pam project cyberark-import pvwa.company.com --dry-run --output import.json --in
 # Filter specific safes
 pam project cyberark-import pvwa.company.com --safes "Production,Staging" --exclude-safes "Archive*"
 
+# Import into Nested Share Folders (folders, records, rotation, PAM config)
+pam project cyberark-import pvwa.company.com --name "CyberArk Migration" --gateway "My Gateway" --nsf
+
 # Extend existing project
 pam project cyberark-import pvwa.company.com --config <pam-config-uid>
 
@@ -196,7 +204,8 @@ pam project cyberark-cleanup --name "CyberArk Migration" --dry-run
 | `--name`, `-n` | Project name |
 | `--config`, `-c` | Extend existing PAM config UID |
 | `--gateway`, `-g` | Gateway name or UID |
-| `--folder-mode` | flat, exact, ksm (default) |
+| `--folder-mode` | flat, exact, ksm, safe (default: safe) |
+| `--nsf` | Create project folders/records/PAM config in Nested Share Folders |
 | `--safes` | Include only these safes (comma/glob) |
 | `--exclude-safes` | Exclude safes (comma/glob) |
 | `--list-safes` | List safes and exit |
@@ -270,5 +279,3 @@ Before building the import JSON, the importer warns about:
 - Rate limit handling: automatic retry on HTTP 429 with exponential backoff
 - Pagination cap: MAX_FETCH_RECORDS (50,000) prevents OOM attacks
 ---
-
-

keepercommander/__init__.py

@@ -10,4 +10,4 @@
 # Contact: commander@keepersecurity.com
 #
 
-__version__ = '18.1.1'
+__version__ = '18.1.2'

keepercommander/command_categories.py

@@ -83,7 +83,8 @@
     # Service Mode REST API
     'Service Mode REST API': {
         'service-create', 'service-add-config', 'service-start', 'service-stop', 'service-status',
-        'service-config-add', 'service-docker-setup', 'slack-app-setup', 'teams-app-setup',
+        'service-config-add', 'service-docker-setup', 'terraform-app-setup',
+        'slack-app-setup', 'teams-app-setup',
         'sailpoint-app-setup', 'gchat-app-setup'
     },
 

keepercommander/commands/discoveryrotation.py

@@ -2566,6 +2566,7 @@ class PamConfigurationEditMixin(RecordEditMixin):
     PAM_CONFIG_RECORD_TYPES = frozenset({
         'pamAwsConfiguration', 'pamAzureConfiguration', 'pamGcpConfiguration',
         'pamDomainConfiguration', 'pamNetworkConfiguration', 'pamOciConfiguration',
+        'pamGitHubConfiguration',
     })
     PAM_RESOURCE_RECORD_TYPES = frozenset({
         'pamDatabase', 'pamDirectory', 'pamMachine', 'pamRemoteBrowser',
@@ -2758,7 +2759,7 @@ def parse_properties(self, params, record, **kwargs):  # type: (KeeperParams, va
                 extra_properties.append(f'text.pamGitHubId={github_id}')
             personal_access_token = kwargs.get('personal_access_token')
             if personal_access_token:
-                extra_properties.append(f'secret.personalAccessToken={personal_access_token}')
+                extra_properties.append(f'secret.pamGitHubPersonalAccessToken={personal_access_token}')
             github_base_url = kwargs.get('github_base_url')
             if github_base_url:
                 extra_properties.append(f'text.pamGitHubBaseUrl={github_base_url}')
@@ -2870,7 +2871,8 @@ def parse_properties(self, params, record, **kwargs):  # type: (KeeperParams, va
     # Fields that the backend previously required but now treats as optional for pamAzureConfiguration.
     AZURE_OPTIONAL_FIELDS = frozenset({'clientId', 'clientSecret'})
 
-    def verify_required(self, record):  # type: (vault.TypedRecord) -> None
+    def verify_required(self, record, command=''):  # type: (vault.TypedRecord, str) -> None
+        missing_fields = []
         for field in record.fields:
             if field.required:
                 if len(field.value) == 0:
@@ -2882,10 +2884,15 @@ def verify_required(self, record):  # type: (vault.TypedRecord) -> None
                           and field.label in self.AZURE_OPTIONAL_FIELDS):
                         pass
                     else:
-                        self.warnings.append(f'Empty required field: "{field.get_field_name()}"')
+                        missing_fields.append(field.get_field_name())
         for custom in record.custom:
             if custom.required:
                 custom.required = False
+        if missing_fields:
+            if len(missing_fields) == 1:
+                raise CommandError(command, f'Empty required field: "{missing_fields[0]}"')
+            fields_text = ', '.join(f'"{x}"' for x in missing_fields)
+            raise CommandError(command, f'Empty required fields: {fields_text}')
 
 
 class PAMConfigurationNewCommand(Command, PamConfigurationEditMixin):
@@ -2986,7 +2993,7 @@ def execute(self, params, **kwargs):
         if not gateway_uid:
             logging.warning(f'Gateway "{gw_name}" not found.')
 
-        self.verify_required(record)
+        self.verify_required(record, command='pam-config-new')
 
         create_pam_configuration_in_folder(params, record, shared_folder_uid, command='pam-config-new')
 
@@ -3119,7 +3126,7 @@ def execute(self, params, **kwargs):
             orig_admin_cred_ref = value.get('adminCredentialRef') or ''
 
         self.parse_properties(params, configuration, config_edit=True, **kwargs)
-        self.verify_required(configuration)
+        self.verify_required(configuration, command='pam-config-edit')
 
         update_pam_record(params, configuration, command='pam-config-edit')
 
@@ -3812,6 +3819,10 @@ def execute(self, params, **kwargs):
             # Find and load email config to validate provider and dependencies
             try:
                 config_uid = find_email_config_record(params, self.email_config)
+                if not config_uid:
+                    raise CommandError(
+                        'pam action rotate',
+                        f'Email configuration "{self.email_config}" not found')
                 email_config_obj = load_email_config_from_record(params, config_uid)
 
                 # Check if required dependencies are installed for this provider

keepercommander/commands/discoveryrotation_v1.py

@@ -970,7 +970,8 @@ def parse_properties(self, params, record, **kwargs):  # type: (KeeperParams, va
         if extra_properties:
             self.assign_typed_fields(record, [RecordEditMixin.parse_field(x) for x in extra_properties])
 
-    def verify_required(self, record):    # type: (vault.TypedRecord) -> None
+    def verify_required(self, record, command=''):    # type: (vault.TypedRecord, str) -> None
+        missing_fields = []
         for field in record.fields:
             if field.required:
                 if len(field.value) == 0:
@@ -981,10 +982,15 @@ def verify_required(self, record):    # type: (vault.TypedRecord) -> None
                             'tz': 'Etc/UTC',
                         }]
                     else:
-                        self.warnings.append(f'Empty required field: "{field.get_field_name()}"')
+                        missing_fields.append(field.get_field_name())
         for custom in record.custom:
             if custom.required:
                 custom.required = False
+        if missing_fields:
+            if len(missing_fields) == 1:
+                raise CommandError(command, f'Empty required field: "{missing_fields[0]}"')
+            fields_text = ', '.join(f'"{x}"' for x in missing_fields)
+            raise CommandError(command, f'Empty required fields: {fields_text}')
 
 
 class PAMConfigurationNewCommand(Command, PamConfigurationEditMixin):
@@ -1037,7 +1043,7 @@ def execute(self, params, **kwargs):
         if not shared_folder_uid:
             raise CommandError('pam-config-new', '--shared_folder parameter is required to create a PAM configuration')
 
-        self.verify_required(record)
+        self.verify_required(record, command='pam-config-new')
 
         pam_configuration_create_record_v6(params, record, shared_folder_uid)
 
@@ -1126,7 +1132,7 @@ def execute(self, params, **kwargs):
             orig_shared_folder_uid = value.get('folderUid') or ''
 
         self.parse_properties(params, configuration, **kwargs)
-        self.verify_required(configuration)
+        self.verify_required(configuration, command='pam-config-edit')
 
         record_management.update_record(params, configuration)
 

keepercommander/commands/email_commands.py

@@ -151,9 +151,28 @@
 # Helper Functions
 # =============================================================================
 
+def _is_owned_email_config_record(params: KeeperParams, record_uid: str) -> bool:
+    """True if the current account owns this record.
+
+    Uses ``record_owner_cache`` first, then ``meta_data_cache`` from
+    recordMetaData (server-asserted). Unknown ownership fails closed.
+    """
+    owner = (params.record_owner_cache or {}).get(record_uid)
+    if owner and owner.owner:
+        return True
+    meta = (getattr(params, 'meta_data_cache', None) or {}).get(record_uid) or {}
+    return bool(meta.get('owner'))
+
+
 def find_email_config_record(params: KeeperParams, name: str) -> Optional[str]:
     """
-    Find email config record by name.
+    Find an owned email config record by name.
+
+    Only records owned by the current account are eligible. Shared or
+    non-owned records (even with a matching title and ``__email_config__``
+    marker) are ignored so SMTP/provider settings cannot be supplied by
+    another user. A missing or unknown ownership entry is treated the same
+    way (not owned by the current account, or ownership unknown).
 
     Args:
         params: KeeperParams session
@@ -169,15 +188,26 @@ def find_email_config_record(params: KeeperParams, name: str) -> Optional[str]:
         if record.record_type != 'login':
             continue
 
-        # Check if this is an email config by looking for custom field
         try:
             record_dict = vault_extensions.extract_typed_record_data(record)
             custom_fields = record_dict.get('custom', [])
-            for field in custom_fields:
-                if field.get('type') == 'text' and field.get('label') == '__email_config__':
-                    if record.title == name:
-                        return record_uid
-        except:
+            is_email_config = any(
+                field.get('type') == 'text' and field.get('label') == '__email_config__'
+                for field in custom_fields
+            )
+            if not is_email_config:
+                continue
+            if record.title != name:
+                continue
+            if not _is_owned_email_config_record(params, record_uid):
+                logging.warning(
+                    'Ignoring email configuration "%s" (%s): '
+                    'not owned by the current account (or ownership unknown)',
+                    name, record_uid)
+                continue
+            return record_uid
+        except Exception as e:
+            logging.debug('Skipping record %s: %s', record_uid, e)
             continue
 
     return None
@@ -591,13 +621,17 @@ def execute(self, params: KeeperParams, **kwargs):
 
 
 class EmailConfigListCommand(Command):
-    """List all email configurations."""
+    """List owned email configurations."""
 
     def get_parser(self):
         return email_config_list_parser
 
     def execute(self, params: KeeperParams, **kwargs):
-        """Execute email-config list command."""
+        """Execute email-config list command.
+
+        Only configurations owned by the current account are listed, matching
+        ``find_email_config_record`` eligibility used by test/delete/--send-email.
+        """
         configs = []
 
         # Find all email config records
@@ -629,13 +663,22 @@ def execute(self, params: KeeperParams, **kwargs):
                         if values:
                             from_address = values[0]
 
-                if is_email_config:
-                    configs.append({
-                        'name': record.title,
-                        'record_uid': record_uid,
-                        'provider': provider or 'unknown',
-                        'from_address': from_address or ''
-                    })
+                if not is_email_config:
+                    continue
+
+                if not _is_owned_email_config_record(params, record_uid):
+                    logging.debug(
+                        'Skipping email configuration "%s" (%s) in list: '
+                        'not owned by the current account (or ownership unknown)',
+                        record.title, record_uid)
+                    continue
+
+                configs.append({
+                    'name': record.title,
+                    'record_uid': record_uid,
+                    'provider': provider or 'unknown',
+                    'from_address': from_address or ''
+                })
             except Exception as e:
                 logging.debug(f'Error loading record {record_uid}: {e}')
                 continue

keepercommander/commands/nested_share_folder/display_commands.py

@@ -87,7 +87,10 @@ def execute(self, params, **kwargs):
 class NestedShareGetCommand(Command):
     """Show details of a Nested Share Record or folder."""
 
-    _MASKED_TYPES = frozenset({'password', 'secret', 'pinCode', 'pin_code'})
+    _MASKED_TYPES = frozenset({
+        'password', 'secret', 'pinCode', 'pin_code', 'oneTimeCode', 'otp',
+        'note', 'json',
+    })
 
     def get_parser(self):
         return nested_share_get_parser
@@ -196,10 +199,19 @@ def _record_detail(self, params, record_uid, verbose, unmask):
         if url_val:
             print('{0:>20s}: {1:<20s}'.format('URL', url_val))
 
-        shown_types = {'login', 'password', 'url'}
-        for f in meta['fields']:
+        self._print_typed_fields(meta['fields'], unmask, skip_types={'login', 'password', 'url'})
+        self._print_typed_fields(meta.get('custom') or [], unmask)
+
+        if meta['notes']:
+            for i, line in enumerate(meta['notes'].split('\n')):
+                print('{0:>21s} {1}'.format('Notes:' if i == 0 else '', line.strip()))
+
+        self._print_record_permissions(params, record_uid, verbose)
+
+    def _print_typed_fields(self, fields, unmask, skip_types=()):
+        for f in fields or []:
             ftype = f.get('type', '')
-            if ftype in shown_types:
+            if ftype in skip_types:
                 continue
             label = f.get('label') or ftype.replace('_', ' ').title()
             values = f.get('value', [])
@@ -216,12 +228,6 @@ def _record_detail(self, params, record_uid, verbose, unmask):
                     dval = str(val)
                 print('{0:>20s}: {1:<s}'.format(label, dval))
 
-        if meta['notes']:
-            for i, line in enumerate(meta['notes'].split('\n')):
-                print('{0:>21s} {1}'.format('Notes:' if i == 0 else '', line.strip()))
-
-        self._print_record_permissions(params, record_uid, verbose)
-
     @staticmethod
     def _extract_field_value(fields, field_type):
         """Extract the first non-empty value for a given field type."""
@@ -247,6 +253,8 @@ def _record_json(self, params, record_uid, verbose, _unmask=False, include_dag=F
             ro['folder'] = meta['folder_location']
         if meta['fields']:
             ro['fields'] = meta['fields']
+        if meta.get('custom'):
+            ro['custom'] = meta['custom']
         if meta['notes']:
             ro['notes'] = meta['notes']
 

keepercommander/commands/nested_share_folder/helpers.py

@@ -705,14 +705,15 @@ def load_record_metadata(params, record_uid):
     """Load record metadata from cache, falling back to the v3 details API.
 
     Returns a dict with keys:
-        ``title``, ``type``, ``fields``, ``notes``,
+        ``title``, ``type``, ``fields``, ``custom``, ``notes``,
         ``revision``, ``version``, ``folder_location``
     """
     from ... import nested_share_folder as _nsf
 
     title = record_uid
     rec_type = ''
     fields = []
+    custom = []
     notes = ''
     revision = 0
     version = 0
@@ -724,7 +725,12 @@ def load_record_metadata(params, record_uid):
             dj = data_obj['data_json']
             title    = dj.get('title', record_uid)
             rec_type = dj.get('type', '')
-            fields   = dj.get('fields', [])
+            fields   = dj.get('fields') or []
+            custom   = dj.get('custom') or []
+            if not isinstance(fields, list):
+                fields = []
+            if not isinstance(custom, list):
+                custom = []
             notes    = dj.get('notes', '') or ''
 
     nsf_records = getattr(params, 'nested_share_records', {})
@@ -749,6 +755,7 @@ def load_record_metadata(params, record_uid):
         'title': title,
         'type': rec_type,
         'fields': fields,
+        'custom': custom,
         'notes': notes,
         'revision': revision,
         'version': version,