Skip to content

Commit

Permalink
Version 141
Browse files Browse the repository at this point in the history
  • Loading branch information
@hydrusnetwork
hydrusnetwork committed Dec 25, 2014
1 parent 5127f7b commit b7edfb9
Show file tree
Hide file tree
Showing 14 changed files with 1,054 additions and 218 deletions.
23 changes: 23 additions & 0 deletions help/changelog.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,29 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 141</h3></li>
<ul>
<li>combined mappings are no longer calculated and stored</li>
<li>recalculate combined mappings obviously removed from database menu</li>
<li>combined mappings no longer have to be recalculated on a service deletion; the accompanying 'this could take a long time' warning dialog is gone as well</li>
<li>combined mappings autocomplete counts are now calculated on-the-fly</li>
<li>combined mappings tag queries are now performed on-the-fly</li>
<li>combined mappings namespace queries are now performed on-the-fly</li>
<li>combined mappings zero/non-zero tag count queries are now performed on-the-fly</li>
<li>combined mappings regular tag count queries are now performed on-the-fly</li>
<li>corrected some logic with regular tag count queries, I think</li>
<li>autocomplete tag cache dirties and culls itself more efficiently on tag update</li>
<li>autocomplete tag cache dirties and culls itself more efficiently on file import/delete</li>
<li>removed a couple of non-useful AMP tests that were breaking due to a previous change in connection code</li>
<li>improved how popup messages give the gui permission to update; hopefully the gui will lock up less when big jobs with popups are happening</li>
<li>improved some misc timing logic</li>
<li>improved some repo sync timing logic</li>
<li>added simple emergency throttling to the repo sync daemon when the CPU/HDD is getting hammered</li>
<li>improved some repo sync text number-grammar and timing</li>
<li>added sankaku channel booru, including flash</li>
<li>the booru downloading code that discovers image urls is more flexible</li>
<li>improved my job pause/cancel logic so paused jobs, when cancelled/completed, will report themselves as no longer paused (this was affecting the downloader page, with paused jobs not dismissing themselves on a subsequent cancel)</li>
</ul>
<li><h3>version 140</h3></li>
<ul>
<li>if a repository or subscription popup message has nothing to report, it will dismiss itself</li>
Expand Down
13 changes: 12 additions & 1 deletion include/ClientConstants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1051,6 +1051,17 @@ def GetNamespaces( self ): return self._tag_classnames_to_namespaces.values()

DEFAULT_BOORUS[ 'tbib' ] = Booru( name, search_url, search_separator, advance_by_page_num, thumb_classname, image_id, image_data, tag_classnames_to_namespaces )

name = 'sankaku chan'
search_url = 'https://chan.sankakucomplex.com/?tags=%tags%&page=%index%'
search_separator = '+'
advance_by_page_num = True
thumb_classname = 'thumb'
image_id = 'highres'
image_data = None
tag_classnames_to_namespaces = { 'tag-type-general' : '', 'tag-type-character' : 'character', 'tag-type-copyright' : 'series', 'tag-type-artist' : 'creator' }

DEFAULT_BOORUS[ 'sankaku chan' ] = Booru( name, search_url, search_separator, advance_by_page_num, thumb_classname, image_id, image_data, tag_classnames_to_namespaces )

class LocalBooruCache( object ):

def __init__( self ):
Expand Down Expand Up @@ -2708,7 +2719,7 @@ def DAEMONWaterfall( self ):

if HC.GetNowPrecise() - last_paused > 0.005:

time.sleep( 0.0001 )
time.sleep( 0.00001 )

last_paused = HC.GetNowPrecise()

Expand Down
2 changes: 1 addition & 1 deletion include/ClientController.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,7 @@ def WaitUntilGoodTimeToUseGUIThread( self ):

if HC.shutdown: raise Exception( 'Client shutting down!' )
elif HC.pubsub.NoJobsQueued() and not HC.currently_doing_pubsub: return
else: time.sleep( 0.0001 )
else: time.sleep( 0.00001 )



Expand Down
302 changes: 140 additions & 162 deletions include/ClientDB.py
View file

Large diffs are not rendered by default.

10 changes: 0 additions & 10 deletions include/ClientGUI.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -845,7 +845,6 @@ def database():
menu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'vacuum_db' ), p( '&Vacuum' ), p( 'Rebuild the Database.' ) )
menu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'delete_orphans' ), p( '&Delete Orphan Files' ), p( 'Go through the client\'s file store, deleting any files that are no longer needed.' ) )
menu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'delete_service_info' ), p( '&Clear Service Info Cache' ), p( 'Delete all cache service info, in case it has become desynchronised.' ) )
menu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'regenerate_combined_mappings' ), p( '&Regenerate Combined Mappings' ), p( 'Delete and rebuild the combined cache of all your services\' mappings.' ) )
menu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'regenerate_thumbnails' ), p( '&Regenerate All Thumbnails' ), p( 'Delete all thumbnails and regenerate from original files.' ) )
menu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'file_integrity' ), p( '&Check File Integrity' ), p( 'Review and fix all local file records.' ) )
menu.AppendSeparator()
Expand Down Expand Up @@ -1453,14 +1452,6 @@ def _RefreshStatusBar( self ):
self._statusbar.SetStatusText( self._statusbar_db_locked, number = 3 )


def _RegenerateCombinedMappings( self ):

with ClientGUIDialogs.DialogYesNo( self, 'Are you sure you want to regenerate the combined mappings? This can take a long time.' ) as dlg:

if dlg.ShowModal() == wx.ID_YES: HC.app.Write( 'regenerate_combined_mappings' )



def _RegenerateThumbnails( self ):

text = 'This will rebuild all your thumbnails from the original files. You probably only want to do this if you experience thumbnail errors. If you have a lot of files, it will take some time. A popup message will show its progress.'
Expand Down Expand Up @@ -1909,7 +1900,6 @@ def EventMenu( self, event ):

if page is not None: page.RefreshQuery()

elif command == 'regenerate_combined_mappings': self._RegenerateCombinedMappings()
elif command == 'regenerate_thumbnails': self._RegenerateThumbnails()
elif command == 'restore_database': HC.app.RestoreDatabase()
elif command == 'review_services': self._ReviewServices()
Expand Down
14 changes: 0 additions & 14 deletions include/ClientGUIDialogsManage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5116,20 +5116,6 @@ def EventRemove( self, event ):

( service_key, service_type, name, info ) = service_panel.GetInfo()

if service_type == HC.TAG_REPOSITORY:

text = 'Deleting a tag service is a potentially very expensive operation.'
text += os.linesep * 2
text += 'If you have millions of tags, it could take twenty minutes or more, during which time your database will be locked.'
text += os.linesep * 2
text += 'Are you sure you want to delete ' + name + '?'

with ClientGUIDialogs.DialogYesNo( self, text ) as dlg:

if dlg.ShowModal() != wx.ID_YES: return



self._edit_log.append( HC.EditLogActionDelete( service_key ) )

services_listbook.DeleteCurrentPage()
Expand Down
8 changes: 6 additions & 2 deletions include/HydrusConstants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
# Misc

NETWORK_VERSION = 15
SOFTWARE_VERSION = 140
SOFTWARE_VERSION = 141

UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )

Expand Down Expand Up @@ -1989,6 +1989,8 @@ def DeleteVariable( self, name ):
if name in self._variables: del self._variables[ name ]


time.sleep( 0.00001 )


def Finish( self ): self._done.set()

Expand Down Expand Up @@ -2016,7 +2018,7 @@ def IsDone( self ): return shutdown or self._done.is_set()

def IsPausable( self ): return self._pausable and not self.IsDone()

def IsPaused( self ): return self._paused.is_set()
def IsPaused( self ): return self._paused.is_set() and not self.IsDone()

def IsWorking( self ): return self.IsBegun() and not self.IsDone()

Expand All @@ -2038,6 +2040,8 @@ def SetVariable( self, name, value ):

with self._variable_lock: self._variables[ name ] = value

time.sleep( 0.00001 )


def WaitOnPause( self ):

Expand Down
71 changes: 55 additions & 16 deletions include/HydrusDownloading.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import json
import os
import pafy
import re
import threading
import time
import traceback
Expand Down Expand Up @@ -198,6 +199,16 @@ def starts_with_classname( classname ): return classname is not None and classna

thumbnails = soup.find_all( class_ = starts_with_classname )

# this is a sankaku thing
popular_thumbnail_parent = soup.find( id = 'popular-preview' )

if popular_thumbnail_parent is not None:

popular_thumbnails = popular_thumbnail_parent.find_all( class_ = starts_with_classname )

thumbnails = thumbnails[ len( popular_thumbnails ) : ]


if self._gallery_advance_num is None:

if len( thumbnails ) == 0: self._we_are_done = True
Expand Down Expand Up @@ -241,16 +252,34 @@ def _ParseImagePage( self, html, url_base ):

image = soup.find( id = image_id )

image_url = image[ 'src' ]

if 'sample/sample-' in image_url:
if image is None:

# danbooru resized image
image_string = soup.find( text = re.compile( 'Save this file' ) )

image = soup.find( id = 'image-resize-link' )
image = image_string.parent

image_url = image[ 'href' ]

else:

if image.name == 'img':

image_url = image[ 'src' ]

if 'sample/sample-' in image_url:

# danbooru resized image

image = soup.find( id = 'image-resize-link' )

image_url = image[ 'href' ]


elif image.name == 'a':

image_url = image[ 'href' ]




if image_data is not None:
Expand Down Expand Up @@ -1436,15 +1465,19 @@ def MainLoop( self ):

while not self._controller_job_key.IsDone():

if self._controller_job_key.IsPaused():
while self._controller_job_key.IsPaused():

time.sleep( 0.1 )

self._import_job_key.Pause()
self._import_queue_position_job_key.Pause()
self._import_queue_job_key.Pause()

self._controller_job_key.WaitOnPause()
if HC.shutdown or self._controller_job_key.IsDone(): break


if HC.shutdown or self._controller_job_key.IsDone(): break

with self._lock:

queue_position = self._import_queue_position_job_key.GetVariable( 'queue_position' )
Expand Down Expand Up @@ -1586,14 +1619,16 @@ def __call__( self ):

for downloader in downloaders:

if self._job_key.IsPaused():
while self._job_key.IsPaused():

time.sleep( 0.1 )

self._job_key.SetVariable( 'status', 'paused after ' + HC.u( total_urls_found ) + ' urls' )

self._job_key.WaitOnPause()
if HC.shutdown or self._job_key.IsDone(): break


if self._job_key.IsCancelled(): break
if HC.shutdown or self._job_key.IsDone(): break

self._job_key.SetVariable( 'status', 'found ' + HC.u( total_urls_found ) + ' urls' )

Expand All @@ -1618,14 +1653,16 @@ def __call__( self ):

if len( downloaders ) == 0: break

if self._job_key.IsPaused():
while self._job_key.IsPaused():

time.sleep( 0.1 )

self._job_key.SetVariable( 'status', 'paused after ' + HC.u( total_urls_found ) + ' urls' )

self._job_key.WaitOnPause()
if HC.shutdown or self._job_key.IsDone(): break


if self._job_key.IsCancelled(): break
if HC.shutdown or self._job_key.IsDone(): break


self._job_key.SetVariable( 'status', '' )
Expand Down Expand Up @@ -1705,14 +1742,16 @@ def __call__( self ):



if self._job_key.IsPaused():
while self._job_key.IsPaused():

time.sleep( 0.1 )

self._job_key.SetVariable( 'status', 'paused' )

self._job_key.WaitOnPause()
if HC.shutdown or self._job_key.IsDone(): break


if self._job_key.IsCancelled(): break
if HC.shutdown or self._ob_key.IsDone(): break

thread_time = self._job_key.GetVariable( 'thread_time' )

Expand Down
10 changes: 0 additions & 10 deletions include/ServerDB.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2215,16 +2215,6 @@ def _MakeBackup( self ):

def _UpdateDB( self, version ):

if version == 90:

for ( service_id, options ) in self._c.execute( 'SELECT service_id, options FROM services;' ).fetchall():

options[ 'upnp' ] = None

self._c.execute( 'UPDATE services SET options = ? WHERE service_id = ?;', ( options, service_id ) )



if version == 93:

self._c.execute( 'CREATE TABLE messaging_sessions ( service_id INTEGER REFERENCES services ON DELETE CASCADE, session_key BLOB_BYTES, account_id INTEGER, identifier BLOB_BYTES, name TEXT, expiry INTEGER );' )
Expand Down
50 changes: 50 additions & 0 deletions include/TestHydrusDownloading.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,56 @@ def test_pixiv( self ):
self.assertEqual( info, expected_info )


def test_sankaku( self ):

with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'sankaku_gallery.html' ) as f: sankaku_gallery = f.read()
with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'sankaku_page.html' ) as f: sankaku_page = f.read()

HC.http.SetResponse( HC.GET, 'https://chan.sankakucomplex.com/?tags=animal_ears&page=1', sankaku_gallery )
HC.http.SetResponse( HC.GET, 'https://chan.sankakucomplex.com/post/show/4324703', sankaku_page )

#

downloader = HydrusDownloading.DownloaderBooru( CC.DEFAULT_BOORUS[ 'sankaku chan' ], [ 'animal_ears' ] )

#

gallery_urls = downloader.GetAnotherPage()

expected_gallery_urls = [(u'https://chan.sankakucomplex.com/post/show/4324703',), (u'https://chan.sankakucomplex.com/post/show/4324435',), (u'https://chan.sankakucomplex.com/post/show/4324426',), (u'https://chan.sankakucomplex.com/post/show/4324365',), (u'https://chan.sankakucomplex.com/post/show/4324343',), (u'https://chan.sankakucomplex.com/post/show/4324309',), (u'https://chan.sankakucomplex.com/post/show/4324134',), (u'https://chan.sankakucomplex.com/post/show/4324107',), (u'https://chan.sankakucomplex.com/post/show/4324095',), (u'https://chan.sankakucomplex.com/post/show/4324086',), (u'https://chan.sankakucomplex.com/post/show/4323969',), (u'https://chan.sankakucomplex.com/post/show/4323967',), (u'https://chan.sankakucomplex.com/post/show/4323665',), (u'https://chan.sankakucomplex.com/post/show/4323620',), (u'https://chan.sankakucomplex.com/post/show/4323586',), (u'https://chan.sankakucomplex.com/post/show/4323581',), (u'https://chan.sankakucomplex.com/post/show/4323580',), (u'https://chan.sankakucomplex.com/post/show/4323520',), (u'https://chan.sankakucomplex.com/post/show/4323512',), (u'https://chan.sankakucomplex.com/post/show/4323498',)]

self.assertEqual( gallery_urls, expected_gallery_urls )

#

HC.http.SetResponse( HC.GET, 'https://cs.sankakucomplex.com/data/c5/c3/c5c3c91ca68bd7662f546cc44fe0d378.jpg?4324703', 'image file' )

info = downloader.GetFileAndTags( 'https://chan.sankakucomplex.com/post/show/4324703' )

( temp_path, tags ) = info

with open( temp_path, 'rb' ) as f: data = f.read()

info = ( data, tags )

expected_info = ('image file', [u'character:heinrike prinzessin zu sayn-wittgenstein', u'character:rosalie de hemricourt de grunne', u'2girls', u'alternative costume', u'anal beads', u'animal ears', u'anus', u'ass', u'ass cutout', u'backless panties', u'blonde', u'blue eyes', u'blush', u'braid', u'butt plug', u'butt plug tail', u'cameltoe', u'cat tail', u'cheerleader', u'dildo', u'fake animal ears', u'fang', u'green eyes', u'hands', u'happy', u'heart cutout', u'kneepits', u'long hair', u'looking at viewer', u'multiple girls', u'nekomimi', u'open mouth', u'pantsu', u'spread anus', u'sweat', u'tail', u'tape', u'underwear', u'white panties', u'creator:null (nyanpyoun)', u'series:strike witches'])

self.assertEqual( info, expected_info )

# flash is tricky for sankaku

with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'sankaku_flash.html' ) as f: sankaku_flash = f.read()

HC.http.SetResponse( HC.GET, 'https://chan.sankakucomplex.com/post/show/4318061', sankaku_flash )
HC.http.SetResponse( HC.GET, 'https://cs.sankakucomplex.com/data/48/ce/48cecd707d8a562d47db74d934505f51.swf?4318061', 'swf file' )

temp_path = downloader.GetFile( 'https://chan.sankakucomplex.com/post/show/4318061' )

with open( temp_path, 'rb' ) as f: data = f.read()

self.assertEqual( data, 'swf file' )


def test_booru_e621( self ):

with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'e621_gallery.html' ) as f: e621_gallery = f.read()
Expand Down
4 changes: 2 additions & 2 deletions include/TestHydrusServer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ def test_local_booru( self ):
self._test_basics( host, port )
self._test_local_booru( host, port )


'''
class TestAMP( unittest.TestCase ):
@classmethod
Expand Down Expand Up @@ -777,4 +777,4 @@ def test_message( self ):
[ ( args, kwargs ) ] = result
self.assertEqual( args, ( temp_identifier, temp_name, persistent_identifier, persistent_name, message ) )

'''
Loading

0 comments on commit b7edfb9

Please sign in to comment.