Repository URL to install this package:
|
Version:
0.2.2 ▾
|
PyPxTools
/
pypxtools.py
|
|---|
# -*- coding: utf-8 -*-
from __future__ import print_function
import collections
from collections import OrderedDict
import threading
import os
import inspect
from IPython.display import display
from ipywidgets import widgets
from pypxtools.pxobjects import PxQuery, PxProject, PxContent
import datetime
import pandas as pd
def flatten_list(list_of_lists):
"""
Removes list within lists, makes one flat list instead
:param list_of_lists:
:return:
"""
# implementation taken from 'Christian's answer here
# http://stackoverflow.com/questions/2158395/flatten-an-irregular-list-of-lists-in-python
basestring = (str, bytes)
for el in list_of_lists:
if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
for sub in flatten_list(el):
yield sub
else:
yield el
class ListExport(object):
"""
Class for exporting lists to text file/ as script
"""
# Set constant settings
SAMSTAGE_COMMENT = """
# SAM Staging command (_input_file_ is this file):
# /opt/SUNWsamfs/bin/stage -i _input_file_" # cmd path for hsm.geomar.de
"""
def __init__(self, query, out_file=None):
self.query = query
self.out_file = out_file
def get_level_indicator(self, lvl):
return '#' + str(['#' for l in range(lvl)])\
.replace(',', '').replace('[', '')\
.replace(']', '').replace(' ', '').replace("'", "")
def get_project_line(self, prj, content_count=True, sum_size=True):
ret = '# '+prj.name + ' : ' + prj.uuid
if content_count:
ret = ret + ' No. Items: ' + str(prj.get_content_count(include_subdirs=False))\
+ '/' + str(prj.get_content_count(include_subdirs=True))
# TODO implement sumsize
return ret
def get_folder_line(self, folder, level=True, content_count=True, sum_size=True):
ret = folder.name+' : ['+folder.uuid+']'
if level:
ret = self.get_level_indicator(folder.get_level())+' ' + ret
if content_count:
ret = ret + ' No. Items: ' + str(folder.get_content_count(include_subdirs=False))\
+ '/' + str(folder.get_content_count(include_subdirs=True))
if sum_size:
ret = ret + ' Sum file size: ' + str(folder.get_sum_filesize())
return ret
def export_content_summary(self, prj):
self.print_wrapper(self.get_project_line(prj), context=prj)
for folder in prj.get_all_folders_ordered():
self.print_wrapper(self.get_folder_line(folder), context=folder)
def get_content_original_file_path(self, content):
"""
Makes line for the file location of a content item
:param content:
:return:
"""
return content.folder_path
def get_content_original_file_location(self, content):
"""
Makes line for the file location of a content item
:param content:
:return:
"""
#ret = self.get_content_original_file_path(content)
#ret = ret+u'/v.{}.{}'.format(content.version, content.file_name)
ret = content.get_file_path()
return "'"+ret+"'"
def print_content_staging(self, prj):
"""
Outputs appropriate file list for the SAM staging command
:param prj:
:return:
"""
self.print_wrapper(ListExport.SAMSTAGE_COMMENT, context=prj)
self.print_wrapper(self.get_project_line(prj), context=prj)
for folder in prj.get_all_folders_ordered():
for content in prj.get_content_list(include_subdirs=False):
self.print_wrapper(self.get_content_original_file_location(content), context=content)
self.print_wrapper(self.get_folder_line(folder), context=folder)
for content in folder.get_content(include_subdirs=False):
self.print_wrapper(self.get_content_original_file_location(content), context=content)
def print_wrapper(self, message, context=None):
'''
Wrapper for printing service. Includes error handling.
:param message: Message string to print
:return: Context object, e.g. PxFolder, PxContent
'''
# the explicit encoding of the text is absolutely necessary when writing to a file:
# http://stackoverflow.com/questions/9822655/python-unicode-write-to-file-crashes-in-command-line-but-not-in-ide
message = message.encode("utf-8", "replace")
try:
print(message, file=self.out_file)
except UnicodeEncodeError as e:
uuid = ''
if context is not None:
uuid = context.uuid
else:
context = 'None'
print('could not print message for context '+uuid)
class PrjSelectWdgt(widgets.Dropdown):
"""
Custom iPywidget to select project
"""
def __init__(self, px_query, *args, **kwargs):
super(widgets.Dropdown, self).__init__(*args, **kwargs)
self.query = px_query
# get list w/ projects fro ProxSys Server. This will display login prompt
pl = self.query.get_project_list()
# make a dictionary w/ prj name as key, prj uuid as value
pl_dict = pl.set_index('pname')['projectId'].to_dict()
pl_dict_ordered = OrderedDict() # we want orderd dict to assure alphabetical ordering
# simply calling pl_dict_ordered = OrderedDict(pl_dict) produces unsorted dict :/
# --> order explicitly by order added to OD
for p_name in pl.pname.values: # projects are returned ordered alphabetically in list
# add entry for each prj to ordered dict
pl_dict_ordered[p_name] = pl_dict[p_name]
# set ordered dict as data here
self.prj_dict = pl_dict_ordered
# set dict as options
self.options = self.prj_dict
# pre-select Testprojekt
pid_testprj = pl.query("pname == 'Testprojekt'").iloc[0].projectId
self._prj = self.query.get_project(pid_testprj)
self.value = pid_testprj
def selection_changed(change):
self._prj = None
self.observe(selection_changed, names='value')
@property
def prj(self):
if self._prj is None:
self._prj = self.query.get_project(self.value)
return self._prj
class FolderSelectWdgt(widgets.SelectMultiple):
"""
Custom iPyWidget to display selection list
"""
def __init__(self, show_stats=True, *args, **kwargs):
super(widgets.SelectMultiple, self).__init__(*args, **kwargs)
self.prj = None
self.layout.width = '80%'
self.layout.height= '300px !important'
def set_prj(self, px_project):
self.prj = px_project
self.set_folders(self.prj.get_all_folders_ordered())
# select 1st elemet
self.value = (list(self.options.items())[0][1],)
def set_folders(self, folder_list):
options = OrderedDict()
options[self._format_item(self.prj)] = 1
for f in folder_list:
folder_name = self._format_item(f)
options[folder_name] = f
self.options = options
def _format_item(self, item):
ctd_count = item.get_content_count(include_subdirs=False)
lvl_idc = u'\u2517' + u'\u2501' * item.get_level()
item_name = u'{idc} {name} [{uuid}] ({count} files)'.format(
idc=lvl_idc,
name=item.name,
uuid=item.uuid,
count=ctd_count)
return item_name
def get_selected_content(self, progress_bar=None):
ctd = []
prj_id = self.prj.uuid
for item in self.value:
if item == 1: # Project Root entry
folder_id = 'root'
else:
folder_id = item.uuid
ctd.extend(self.prj.query.get_content_chunky(folder_id=folder_id, prj_id=prj_id, progress_bar=progress_bar))
return ctd
class ContentSelect(widgets.Box):
"""
Class w/ dropdown for selecting project, selection list for selecting a folder and a progress bar
indicating folder change in progress.
Implements methods for getting selected project, selected folder(s) and content below selected elements
"""
def __init__(self, px_query, hide=False, *args, **kwargs):
"""
Create and display widgets for project/ folder/ content selection
:param px_query:
:param hide: Toggle if widget is displayed on init or not. Default is to not hide
"""
super(widgets.Box, self).__init__(width='80%', *args, **kwargs)
self.layout.display = 'flex'
self.layout.flex_flow = 'column'
self.layout.align_items = 'stretch'
# the widgets in here
# 1. progress bar shown when selection of project changes and folder list is updated
self.progress = widgets.FloatProgress(description='Getting folders', width='80%')
# 2. dropdown menue for project selection
self.prj_select = PrjSelectWdgt(px_query, description='Select Project:')
# register change listener
self.prj_select.observe(self._prj_selection_change, names='value')
# 3. multi selectionlist w/ folders of project
self.fldr_select = FolderSelectWdgt(show_stats=True, width='80%')
self._prj_selection_change(None) # update folder list to preset prj
if not hide:
display(self.prj_select)
display(self.fldr_select)
display(self.progress)
def _prj_selection_change(self, change):
"""
Called when project selection changes. Updates UI
:param change:
:return:
"""
# Indicate change in progress:
# hide folder selection widget
self.fldr_select.visible = False
self.progress.value=0
# show progress
self.progress.visible = True
# Getting the content count from ProxSys takes the most time. Since the count is cached, getting it once
# will speed up process dramatically the next time around. For this reason, get all counts for all folders
# while progress bar is showing:
# get all folders
folders = self.prj_select.prj.get_all_folders()
self.progress.min = 0
self.progress.max = len(folders)
for folder in folders:
# get content count for folder
folder.get_content_count(include_subdirs=False)
# update progress bar
self.progress.value += 1
# now content count is cached, next call will be quick
# update folder select widget
self.fldr_select.set_prj(self.prj_select.prj)
# hide progress bar
self.progress.visible = False
# show folder select widget
self.fldr_select.visible = True
def get_selected_project(self):
"""
returns project selected in dropdown menue
:return:
"""
return self.prj_select.prj
def get_selected_content(self):
"""
Retuns list with content below selected folder(s)
:return:
"""
self.progress.visible=True
ret = self.fldr_select.get_selected_content(self.progress)
self.progress.visible=False
return ret
def get_selected_folders(self):
"""
Returns list with selected folder(s)
:return:
"""
return self.fldr_select.value
class CtdMoveWdgt(widgets.Box):
"""
Small widget for moving content inside a project and for moving or copying content between projects
Includes progress bar and result list
"""
def __init__(self, px_query:PxQuery, cs_source: ContentSelect = None, cs_target: ContentSelect = None, log_file = None, *args, **kwargs):
"""
"""
super(widgets.Box, self).__init__(*args, **kwargs)
self.layout.display = 'flex'
self.layout.flex_flow = 'column'
self.layout.align_items = 'stretch'
self.query = px_query
self.log_file = log_file
self.log_str = ''
self.cs_source = cs_source
if not self.cs_source:
self.cs_source = ContentSelect(px_query=self.query)
self.cs_target = cs_target
if not self.cs_target:
self.cs_target = ContentSelect(px_query=self.query)
self.do_button = widgets.Button(description='START',
disabled=False,
button_style='', # 'success', 'info', 'warning', 'danger' or ''
tooltip='Start Copy/Move',
icon='check')
self.do_button.on_click(self.do_copy)
self.pxprj_src = None
self.pxprj_target = None
self.contents = []
self.target_folder = None
self.progress = widgets.FloatProgress(value=0,
min=0.0,
max=len(self.contents),
step=1.0,
description='Moving content:')
self.textbox = widgets.Textarea(
description='Processed items:',
value='Filename : Statuscode \n')
self.textbox.layout.width = '80%'
self.children = [widgets.HBox(children=[self.cs_source, self.cs_target]),self.do_button, self.textbox, self.progress]
def do_copy(self, button):
self.pxprj_src = self.cs_source.get_selected_project()
self.pxprj_target = self.cs_target.get_selected_project()
if self.pxprj_src is self.pxprj_target:
self.pxprj_target = None
self.contents = self.cs_source.get_selected_content()
self.target_folder = self.cs_target.get_selected_folders()[0]
if not self.target_folder:
self.target_folder = 'root'
if self.pxprj_target:
action = 'COPYING'
prj_target = self.pxprj_target.name
else:
action = 'MOVING'
prj_target = self.pxprj_src.name
self.log_str = f"""
---------------------------
{datetime.datetime.now()}: {action} content
FROM
project {self.pxprj_src.name}
{self.cs_source.get_selected_folders()}
TO
project {prj_target}
{self.target_folder}
run by {self.query.u_name}
--
"""
if self.pxprj_target: # target prj set -> copy between projects
self.copy_content()
else: # no target prj -> move in same prj
self.move_content()
if self.log_file:
try:
with open(self.log_file, 'a+') as out_file:
out_file.write(self.log_str)
except:
print('ERROR writing to file {}'.format(self.log_file))
def move_content(self):
self.progress.description = 'Moving content'
for ctd in self.contents:
status = None
if not self.pxprj_target: # move within project
status = self.query.move_content_within_project(content_id=ctd.uuid, project_id=self.pxprj_src.uuid,
target_folder_id=self.target_folder.uuid)
else: # move between projects
status = self.query.move_content_between_projects(content_id=ctd.uuid,
source_prj_id=self.pxprj_src.uuid,
target_prj_id=self.pxprj_target.uuid,
target_folder_id=self.target_folder.uuid)
self._update_progress(status, ctd)
def copy_content(self):
if not self.pxprj_target: # can only move in same project
self.move_content()
else:
self.progress.description = 'Copying content'
for ctd in self.contents:
status = self.query.copy_content(ctd.uuid, self.pxprj_target.uuid, self.target_folder.uuid)
self._update_progress(status, ctd)
def _update_progress(self, status, ctd):
self.progress.value += 1
self.textbox.value = self.textbox.value + ctd.file_name + ' : ' + str(status) + '\n'
self.log_str = self.log_str + f"{ctd.uuid}:{ctd.file_name} : {status}\n"
class MetadataBulkEditWdgt(widgets.Box):
"""
Small widget for setting a metadata value on a list of content items inside a project.
Includes progress bar.
"""
def __init__(self, px_query:'PxQuery', content_select:ContentSelect, md_fieldname:str=None, chunk_size:int=100, *args, **kwargs):
"""
Creates a widget for setting a metadata value on a list of content items inside a project. Includes progress bar.
:param px_query: query object used for communication with ProxSys
:param content_select: ContentSelect widget. Used to get selected project and content.
:param md_fieldname: Metadata Field name. If set, metatdata field matching this name will be preselected
:param chunk_size: setting of metadata is performed on list of contents. This parameter determines if setting
should be done in portions of this list and how big these portions should be. Small chunk size means
progress bar is updated more frequently, larger chunk size means less calls to ProxSys server.
"""
super(widgets.Box, self).__init__(*args, **kwargs)
self.layout.display = 'flex'
self.layout.flex_flow = 'column'
self.layout.align_items = 'stretch'
self.query = px_query
self.cs = content_select
self.chunk_size=chunk_size
self.field_select = PxMetadataFieldSelect(self.query, self.cs, md_fieldname)
self.md_value = widgets.Text(description='Enter metadata value:')
self.do_button = widgets.Button(description='START',
disabled=False,
button_style='', # 'success', 'info', 'warning', 'danger' or ''
tooltip='Set metadata value',
icon='check')
self.do_button.on_click(self.do_button_callback)
self.progress = widgets.FloatProgress(value=0,
min=0.0,
# max=len(self.contents),
step=1.0,
description='Writing metadata:')
self.children = [self.field_select, self.md_value, self.do_button, self.progress]
def do_button_callback(self, button):
self.do_set_metadata()
def do_set_metadata(self):
ctd = self.cs.get_selected_content()
self.query.post_bulk_metadata_edit_chunky(ctd, self.field_select.value, self.md_value.value,
chunk_size=self.chunk_size, progress_bar=self.progress)
class PxMetaToExif(widgets.VBox):
""" Widgets for generating exiftool-commands for writing ProxSys metadata to image
"""
EXIF_BASE_CMD = "exiftool -{exif_tag}='{px_metadata_value}' '{px_file_path_media}'"
def __init__(self, px_query:'PxQuery', content_select:ContentSelect, md_fieldname:str=None, *args, **kwargs):
"""
Creates a widget for generating exiftool-commands for writing ProxSys metadata to image.
:param px_query: query object used for communication with ProxSys
:param content_select: ContentSelect widget. Used to get selected project and content.
:param md_fieldname: Metadata Field name. If set, metatdata field matching this name will be preselected
"""
super(widgets.VBox, self).__init__(*args, **kwargs)
self.query = px_query
self.cs = content_select
self.px_md_select = PxMetadataFieldSelect(self.query, self.cs, md_fieldname)
self.exif_md_select = ExifTagSelect()
self.progress = widgets.FloatProgress(value=0,
min=0.0,
# max=len(self.contents),
step=1.0,
description='')
self.children = [self.cs, self.px_md_select, self.exif_md_select, self.progress]
def create_commands(self):
"""Creates exiftool commands for writing PxMetadata to images.
Px Metadata value is queried for each selected content.
Paths in commands are suitable for a script run on media.
"""
self.progress.description = 'Waiting for content ...'
self.progress.value = 0
all_ctd = self.cs.get_selected_content()
self.progress.max = len(all_ctd)
self.progress.description = 'Creating commands'
exif_tag = self.exif_md_select.value
px_md_field = self.px_md_select.value
ret = []
for ctd in all_ctd:
px_metadata_value = ctd.get_metadata_value(px_md_field)
px_file_path_media = ctd.get_file_path()
ret.append(PxMetaToExif.EXIF_BASE_CMD.format(exif_tag=exif_tag, px_metadata_value=px_metadata_value, px_file_path_media=px_file_path_media))
# update progress bar
self.progress.value += 1
return ret
class ExifTagSelect(widgets.Dropdown):
"""Widget for selecting tag name from known exiftool tags
List of tags used :https://sno.phy.queensu.ca/~phil/exiftool/TagNames/EXIF.html
"""
TAG_LIST_FILE_PATH = os.path.join('resources', 'exif_tags.json')
def __init__(self, *args, **kwargs):
style = {'description_width': 'initial'}
layout = widgets.Layout(width='80%')
super(widgets.Dropdown, self).__init__(description='Select Exif/Image metadata field:', style=style, layout=layout,
*args, **kwargs)
base_path = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
exif_tags_path = os.path.join(str(base_path), ExifTagSelect.TAG_LIST_FILE_PATH)
tags_df = pd.read_json(exif_tags_path)
self.options = list(tags_df['Tag Name'])
class PxMetadataFieldSelect(widgets.Dropdown):
"""Widget for selecting a proxsys metadata field
Selectable metadata fields ate constrained by project selected in associated ContentSelect,
i.e. only metadata fields configured for the selected project are displayed, options change when
project selection changes.
"""
def __init__(self, px_query:'PxQuery', content_select:ContentSelect, md_fieldname:str=None, *args, **kwargs):
"""
Creates a widget for setting a metadata value on a list of content items inside a project. Includes progress bar.
:param px_query: query object used for communication with ProxSys
:param content_select: ContentSelect widget. Used to get selected project and content.
:param md_fieldname: Metadata Field name. If set, metatdata field matching this name will be preselected
"""
style = {'description_width': 'initial'}
layout = widgets.Layout(width='80%')
super(widgets.Dropdown, self).__init__(description='Select ProxSys metadata field:', style=style, layout=layout,
*args, **kwargs)
self.query = px_query
self.cs = content_select
self.preselection = md_fieldname
self.cs.prj_select.observe(self.prj_change_callback)
self.load_md_fields()
def prj_change_callback(self, change):
self.load_md_fields()
def load_md_fields(self):
opts = self.query.get_prj_metadata_fields(self.cs.get_selected_project())
self.options = opts
# set preselected value
if self.preselection:
old_val = self.value
# try to find preselection in fields for project
new_val = opts.get(self.preselection, old_val)
self.value = new_val
class CtdCompare(widgets.Box):
"""
Widget for comparing contents from two folders
"""
def __init__(self, px_query: 'PxQuery', cs_left: ContentSelect = None, cs_right: ContentSelect = None, log_file = None, *args, **kwargs):
super(widgets.Box, self).__init__(*args, **kwargs)
self.layout.display = 'flex'
self.layout.flex_flow = 'column'
self.layout.align_items = 'stretch'
self.query = px_query
self.log_file = log_file
if cs_left:
self.cs_left = cs_left
else:
self.cs_left = ContentSelect(self.query, hide=True)
if cs_right:
self.cs_right = cs_right
else:
self.cs_right = ContentSelect(self.query, hide=True)
cs_box = widgets.Box()
cs_box.layout.display = 'flex'
cs_box.layout.flex_flow = 'row'
cs_box.layout.align_items = 'stretch'
cs_box.children = [self.cs_left, self.cs_right]
self.do_button = widgets.Button(description='START',
disabled=False,
button_style='', # 'success', 'info', 'warning', 'danger' or ''
tooltip='Start Comparison',
icon='check')
self.do_button.on_click(self.do_comparison)
self.textbox_only_left = widgets.Textarea(
description='only left',
value='')
self.textbox_only_left.layout.width = '80%'
self.textbox_both = widgets.Textarea(
description='in both',
value='')
self.textbox_both.layout.width = '80%'
self.textbox_only_right = widgets.Textarea(
description='only right',
value='')
self.textbox_only_right.layout.width = '80%'
results_box = widgets.HBox(children=[self.textbox_only_left, self.textbox_both, self.textbox_only_right])
self.children = [cs_box, self.do_button, results_box]
self.ctd_left = []
self.left_ids = []
self.ctd_right = []
self.right_ids = []
def get_ctd(self, which):
if which is 'left':
self.ctd_left = self.cs_left.get_selected_content()
self.left_ids = set([ctd.uuid for ctd in self.ctd_left])
else:
self.ctd_right = self.cs_right.get_selected_content()
self.right_ids = set([ctd.uuid for ctd in self.ctd_right])
def do_comparison(self, button):
"""Start comparison """
t1 = threading.Thread(target=self.get_ctd, kwargs={'which': 'left'})
t2 = threading.Thread(target=self.get_ctd, kwargs={'which': 'right'})
t1.start()
t2.start()
t1.join()
t2.join()
self.report_results()
self.ctd_right = []
self.right_ids = []
self.ctd_left = []
self.left_ids = []
def report_results(self):
"""Writes results of comparison to texboxes, log-file"""
self.textbox_only_left.value = ''
self.textbox_only_right.value = ''
self.textbox_both.value = ''
only_left = self.left_ids - self.right_ids
only_right = self.right_ids - self.left_ids
both = self.left_ids & self.right_ids
self.textbox_only_left.value = '\n'.join(only_left)
self.textbox_only_right.value = '\n'.join(only_right)
self.textbox_both.value = '\n'.join(both)
if self.log_file:
now = datetime.datetime.now()
user = self.query.u_name
prj_left = self.cs_left.get_selected_project().name
folder_left = self.cs_left.get_selected_folders()[0]
prj_right = self.cs_right.get_selected_project().name
folder_right = self.cs_right.get_selected_folders()[0]
num_left = len(only_left)
num_right = len(only_right)
num_both = len(both)
report_str = f"""
-----------------------------------------
{now}: Comparison run by {user}
--
LEFT Project:{prj_left}
LEFT Folder: {folder_left}
--
RIGHT Project:{prj_right}
RIGHT Folder: {folder_right}
--
{num_left} items found only left, {num_both} items in both, {num_right} item found only right
--
LEFT ONLY:
{self.textbox_only_left.value}
--
IN BOTH:
{self.textbox_both.value}
--
RIGHT ONLY:
{self.textbox_only_right.value}
-----------------------------------------
"""
try:
with open(self.log_file, 'a+') as out_file:
out_file.write(report_str)
except:
print('ERROR opening file {}'.format(self.log_file))
class CtdManager(widgets.VBox):
"""
Small widget for moving/ copying content and for comparing folder contents
"""
def __init__(self, px_query:'PxQuery', log_file=None, *args, **kwargs):
"""
Creates a widget for managing content
:param px_query: query object used for communication with ProxSys
:param log_file: file name of log file. Set to None fo no logging
"""
super(widgets.VBox, self).__init__(*args, **kwargs)
self.query = px_query
txt_src = widgets.HTML(value='Select <b>SOURCE</b> ----------------------------------------------')
display(txt_src)
cs_src = ContentSelect(self.query)
txt_trgt = widgets.HTML(value='Select <b>TARGET</b> ----------------------------------------------')
display(txt_trgt)
cs_trgt = ContentSelect(self.query)
txt_sep = widgets.HTML(value='---------------------------------------------------------')
display(txt_sep)
cp = CtdCompare(self.query, cs_left=cs_src, cs_right=cs_trgt, log_file=log_file)
cm = CtdMoveWdgt(self.query, cs_source=cs_src, cs_target=cs_trgt, log_file=log_file)
tabs = widgets.Tab(children=[cm, cp])
tabs.set_title(0, 'Copy')
tabs.set_title(1, 'Compare')
display(tabs)
self.children=[txt_src, cs_src, txt_trgt, cs_trgt, txt_sep, tabs]