diff options
| author | fukachan <fukachan> | 2002-07-10 13:57:28 +0000 |
|---|---|---|
| committer | fukachan <fukachan> | 2002-07-10 13:57:28 +0000 |
| commit | 99859a285a95d2f1ae8dd0ca6b9928cd8e22bab3 (patch) | |
| tree | 36910399f57cda208a54f5df16bc8e0b1567dc8f /cpan | |
| parent | f93e2f63c066ca90b7ff85a904ff2fb2d8daf169 (diff) | |
| download | fml8-99859a285a95d2f1ae8dd0ca6b9928cd8e22bab3.tar.gz fml8-99859a285a95d2f1ae8dd0ca6b9928cd8e22bab3.tar.bz2 fml8-99859a285a95d2f1ae8dd0ca6b9928cd8e22bab3.zip | |
Initial revision
Diffstat (limited to 'cpan')
40 files changed, 10720 insertions, 0 deletions
diff --git a/cpan/dist/Unicode-Japanese/Changes b/cpan/dist/Unicode-Japanese/Changes new file mode 100644 index 00000000..7f793fd7 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Changes @@ -0,0 +1,29 @@ +Revision history for Perl extension Unicode::Japanese. + +0.09 Mon Jul 8 16:54:01 JST 2002 + - many fixes (included vulnerability of DoS) + - supports imode's extended EMOJI only with mapped charcters. + (convertion among each model is not supported yet) + - new implementation; convertion utf8<->ucs2 on XS-mode + - can specify PurePerl mode when loading with 'PurePrel' keyword. + +0.08 Tue Jun 18 13:45:18 JST 2002 + - Supports XS. + - Some bugs has been fixed. + +0.03 Mon Sep 17 17:16:01 JST 2001 + - t/getcode.t added. + - Manual updated. (It translated into English.) + - Fixed bug, doesn't work with perl 5.005. + - Fixed bug, $_ is broken when first method call. + +0.02 Wed Aug 1 17:48:33 JST 2001 + - Manual updated. + - Changed the handling of characters which out of SJIS codes' range + in converting from UTF8 to SJIS. + - The time of loading the module was reduced. (with AUTOLOAD function) + - t/outrange.t added. + +0.01 Fri Jul 27 20:51:44 JST 2001 + - BETA Release. + diff --git a/cpan/dist/Unicode-Japanese/Japanese.h b/cpan/dist/Unicode-Japanese/Japanese.h new file mode 100644 index 00000000..2445c87d --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Japanese.h @@ -0,0 +1,99 @@ + +#ifndef UNICODE__JAPANESE +#define UNICODE__JAPANESE + +/* $Id: Japanese.h,v 1.11 2002/06/30 23:12:58 hio Exp $ */ + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "mediate.h" +#include <assert.h> + +#ifdef __cplusplus +#include "str.h" +#endif + +#ifdef TEST +#include "test.h" +#define ONTEST(cmd) cmd +#else +#define ONTEST(cmd) +#endif + +#ifndef EXTERN_C +# ifdef __cplusplus +# define EXTERN_C extern "C" +# else +# define EXTERN_C +# endif +#endif + +#ifndef u_char +#define u_char u_char +typedef unsigned char u_char; +#endif + +/* util */ +#define new_SV_UNDEF() newSVsv(&PL_sv_undef) + +EXTERN_C +{ + /* sjis <=> utf8 (conv.cpp) */ + SV* xs_sjis_utf8(SV* sv_str); + SV* xs_utf8_sjis(SV* sv_str); + + /* getcode (getcode.cpp) */ + SV* xs_getcode(SV* sv_str); + + /* sjis<=>eucjp, sjis<=>jis */ + SV* xs_sjis_eucjp(SV* sv_str); + SV* xs_eucjp_sjis(SV* sv_str); + SV* xs_sjis_jis(SV* sv_str); + SV* xs_jis_sjis(SV* sv_str); + + /* sjis(i-mode)<=>utf8 */ + SV* xs_sjis_imode_utf8(SV* sv_str); + SV* xs_utf8_sjis_imode(SV* sv_str); + + /* sjis(j-sky)<=>utf8 */ + SV* xs_sjis_jsky_utf8(SV* sv_str); + SV* xs_utf8_sjis_jsky(SV* sv_str); + + /* sjis(dot-i)<=>utf8 */ + SV* xs_sjis_doti_utf8(SV* sv_str); + SV* xs_utf8_sjis_doti(SV* sv_str); + + /* ucs_utf8 */ + SV* xs_ucs2_utf8(SV* sv_str); + SV* xs_utf8_ucs2(SV* sv_str); + + /* メモリマップファイル関連 */ + void do_memmap(); + void do_memunmap(); + + /* SJIS <=> UTF8 変換テーブル */ + /* indexは0..0xffff */ + extern unsigned short const* g_u2s_table; + extern unsigned long const* g_s2u_table; + + /* i-mode/j-sky/dot-i絵文字 <=> UTF8 変換テーブル */ + extern unsigned long const* g_ei2u_table; + extern unsigned short const* g_eu2i_table; + extern unsigned long const* g_ej2u_table; + extern unsigned char const* g_eu2j_table; // char [][5] + extern unsigned long const* g_ed2u_table; + extern unsigned short const* g_eu2d_table; + + /* i-mode/j-sky/dot-i絵文字 <=> UTF8 変換テーブルの要素数 */ + /* バイト数でなく要素数 */ + extern int g_ei2u_size; + extern int g_eu2i_size; + extern int g_ej2u_size; + extern int g_eu2j_size; + extern int g_ed2u_size; + extern int g_eu2d_size; +} + + +#endif /* UNICODE__JAPANESE */ diff --git a/cpan/dist/Unicode-Japanese/Japanese.pm b/cpan/dist/Unicode-Japanese/Japanese.pm new file mode 100644 index 00000000..a61fee02 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Japanese.pm @@ -0,0 +1,2809 @@ +package Unicode::Japanese; +# $Id: Japanese_stub.pm,v 1.25 2002/06/30 23:20:17 hio Exp $ + +use strict; +use vars qw($VERSION $PurePerl $xs_loaderror); +$VERSION = '0.09'; + +sub import +{ + my $pkg = shift; + my @na = grep{ !/^PurePerl$/i }@_; + if( @na != @_ ) + { + $PurePerl = 1; + } + if( @na ) + { + use Carp; + croak("invalid parameter (".join(',',@na).")"); + } +} + +sub DESTROY +{ +} + +sub load_xs +{ + #print STDERR "load_xs\n"; + if( $PurePerl ) + { + #print STDERR "PurePerl mode\n"; + $xs_loaderror = 'disabled'; + return; + } + #print STDERR "XS mode\n"; + + my $use_xs; + LoadXS: + { + + #print STDERR "* * bootstrap...\n"; + eval q + { + use strict; + require DynaLoader; + use vars qw(@ISA); + @ISA = qw(DynaLoader); + local($SIG{__DIE__}) = 'DEFAULT'; + Unicode::Japanese->bootstrap($VERSION); + }; + #print STDERR "* * try done.\n"; + #undef @ISA; + if( $@ ) + { + #print STDERR "failed.\n"; + #print STDERR "$@\n"; + $use_xs = 0; + $xs_loaderror = $@; + undef $@; + last LoadXS; + } + #print STDERR "succeeded.\n"; + $use_xs = 1; + eval q + { + #print STDERR "over riding _s2u,_u2s\n"; + do_memmap(); + #print STDERR "memmap done\n"; + END{ do_memunmap(); } + #print STDERR "binding xsubs done.\n"; + }; + if( $@ ) + { + #print STDERR "error on last part of load XS.\n"; + $xs_loaderror = $@; + CORE::die($@); + } + + #print STDERR "done.\n"; + } + + if( $@ ) + { + $xs_loaderror = $@; + CORE::die("Cannot Load Unicode::Japanese either XS nor PurePerl\n$@"); + } + if( !$use_xs ) + { + #print STDERR "no xs.\n"; + eval q + { + sub do_memmap($){} + sub do_memunmap($){} + }; + } + $xs_loaderror = '' if( !defined($xs_loaderror) ); + #print STDERR "load_xs done.\n"; +} + +use vars qw($FH $TABLE $HEADLEN $PROGLEN); + +sub gensym { + package Unicode::Japanese::Symbol; + no strict; + $genpkg = "Unicode::Japanese::Symbol::"; + $genseq = 0; + my $name = "GEN" . $genseq++; + my $ref = \*{$genpkg . $name}; + delete $$genpkg{$name}; + $ref; +} + +sub _init_table { + + if(!defined($HEADLEN)) + { + $FH = gensym; + + my $file = "Unicode/Japanese.pm"; + OPEN: + { + foreach my $path (@INC) + { + my $mypath = $path; + $mypath =~ s#/$##; + if (-f "$mypath/$file") + { + open($FH,"$mypath/$file") || CORE::die; + binmode($FH); + last OPEN; + } + } + CORE::die "Can't find Japanese.pm in \@INC\n"; + } + + local($/) = "\n"; + my $line; + while($line = <$FH>) + { + last if($line =~ m/^__DATA__/); + } + $PROGLEN = tell($FH); + + read($FH, $HEADLEN, 4) + or die "Can't read table. [$!]\n"; + $HEADLEN = unpack('N', $HEADLEN); + read($FH, $TABLE, $HEADLEN) + or die "Can't seek table. [$!]\n"; + $TABLE = eval $TABLE; + if($@) + { + die "Internal Error. [$@]\n"; + } + if(!defined($TABLE)) + { + die "Internal Error.\n"; + } + $HEADLEN += 4; + + # load xs. + load_xs(); + } +} + +sub _getFile { + my $this = shift; + + my $file = shift; + +# print STDERR "_getFile($file, $TABLE->{$file}{offset}, $TABLE->{$file}{length})\n"; + seek($FH, $PROGLEN + $HEADLEN + $TABLE->{$file}{offset}, 0) + or die "Can't seek $file. [$!]\n"; + + my $data; + read($FH, $data, $TABLE->{$file}{length}) + or die "Can't read $file. [$!]\n"; + + $data; +} + +sub new +{ + my $pkg = shift; + my $this = {}; + + if( defined($pkg) ) + { + bless $this, $pkg; + $this->_init_table; + }else + { + bless $this; + } + + if(defined($_[0])) + { + $this->set(@_); + } + + $this; +} + + + +use vars qw(%CHARCODE %ESC %RE); +use vars qw(@J2S @S2J @S2E @E2S @U2T %T2U %S2U %U2S); + +%CHARCODE = ( + UNDEF_EUC => "\xa2\xae", + UNDEF_SJIS => "\x81\xac", + UNDEF_JIS => "\xa2\xf7", + UNDEF_UNICODE => "\x20\x20", + ); + +%ESC = ( + JIS_0208 => "\e\$B", + JIS_0212 => "\e\$(D", + ASC => "\e\(B", + KANA => "\e\(I", + E_JSKY_START => "\e\$", + E_JSKY_END => "\x0f", + ); + +%RE = + ( + ASCII => '[\x00-\x7f]', + EUC_0212 => '\x8f[\xa1-\xfe][\xa1-\xfe]', + EUC_C => '[\xa1-\xfe][\xa1-\xfe]', + EUC_KANA => '\x8e[\xa1-\xdf]', + JIS_0208 => '\e\$\@|\e\$B|\e&\@\e\$B', + JIS_0212 => "\e" . '\$\(D', + JIS_ASC => "\e" . '\([BJ]', + JIS_KANA => "\e" . '\(I', + SJIS_DBCS => '[\x81-\x9f\xe0-\xef\xfa-\xfc][\x40-\x7e\x80-\xfc]', + SJIS_KANA => '[\xa1-\xdf]', + UTF8 => '[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}', + BOM2_BE => '\xfe\xff', + BOM2_LE => '\xff\xfe', + BOM4_BE => '\x00\x00\xfe\xff', + BOM4_LE => '\xff\xfe\x00\x00', + UTF32_BE => '\x00[\x00-\x10][\x00-\xff]{2}', + UTF32_LE => '[\x00-\xff]{2}[\x00-\x10]\x00', + E_IMODE => '\xf8[\x9f-\xfc]|\xf9[\x40-\x49\x72-\x7e\x80-\xb0]', + E_JSKY1 => '[EFG]', + E_JSKY2 => '[\!-z]', + E_DOTI => '\xf0[\x40-\x7e\x80-\xfc]|\xf1[\x40-\x7e\x80-\xd6]|\xf2[\x40-\x7e\x80-\xab\xb0-\xd5\xdf-\xfc]|\xf3[\x40-\x7e\x80-\xfa]|\xf4[\x40-\x4f\x80\x84-\x8a\x8c-\x8e\x90\x94-\x96\x98-\x9c\xa0-\xa4\xa8-\xaf\xb4\xb5\xbc-\xbe\xc4\xc5\xc8\xcc]', + E_JSKY_START => quotemeta($ESC{E_JSKY_START}), + E_JSKY_END => quotemeta($ESC{E_JSKY_END}), + ); + +$RE{E_JSKY} = $RE{E_JSKY_START} + . $RE{E_JSKY1} . $RE{E_JSKY2} . '+' + . $RE{E_JSKY_END}; + +use vars qw($s2u_table $u2s_table); +use vars qw($ei2u $ed2u $ej2u $eu2i $eu2d $eu2j); + +# encode/decode + +use vars qw(%_h2zNum %_z2hNum %_h2zAlpha %_z2hAlpha %_h2zSym %_z2hSym %_h2zKanaK %_z2hKanaK %_h2zKanaD %_z2hKanaD %_hira2kata %_kata2hira); + + + +AUTOLOAD +{ + use strict; + use vars qw($AUTOLOAD); + + #print STDERR "AUTOLOAD... $AUTOLOAD\n"; + + my $save = $@; + my @BAK = @_; + + my $subname = $AUTOLOAD; + $subname =~ s/^Unicode\:\:Japanese\:\://; + + #print "subs..\n",join("\n",keys %$TABLE,''); + + # check + if(!defined($TABLE->{$subname}{offset})) + { + if (substr($AUTOLOAD,-9) eq '::DESTROY') + { + { + no strict; + *$AUTOLOAD = sub {}; + } + $@ = $save; + @_ = @BAK; + goto &$AUTOLOAD; + } + + CORE::die "Undefined subroutine \&$AUTOLOAD called.\n"; + } + if($TABLE->{$subname}{offset} == -1) + { + CORE::die "Double loaded \&$AUTOLOAD. It has some error.\n"; + } + + seek($FH, $PROGLEN + $HEADLEN + $TABLE->{$subname}{offset}, 0) + or die "Can't seek $subname. [$!]\n"; + + my $sub; + read($FH, $sub, $TABLE->{$subname}{length}) + or die "Can't read $subname. [$!]\n"; + + CORE::eval($sub); + if ($@) + { + CORE::die $@; + } + $DB::sub = $AUTOLOAD; # Now debugger know where we are. + + # evaled + $TABLE->{$subname}{offset} = -1; + + $@ = $save; + @_ = @BAK; + goto &$AUTOLOAD; +} + + +1; + +=head1 NAME + +Unicode::Japanese - Japanese Character Encoding Handler + +=head1 SYNOPSIS + +use Unicode::Japanese; + +# convert utf8 -> sjis + +print Unicode::Japanese->new($str)->sjis; + +# convert sjis -> utf8 + +print Unicode::Japanese->new($str,'sjis')->get; + +# convert sjis (imode_EMOJI) -> utf8 + +print Unicode::Japanese->new($str,'sjis-imode')->get; + +# convert ZENKAKU (utf8) -> HANKAKU (utf8) + +print Unicode::Japanese->new($str)->z2h->get; + +=head1 DESCRIPTION + +Module for conversion among Japanese character encodings. + +=head2 FEATURES + +=over 2 + +=item * + +The instance stores internal strings in UTF-8. + +=item * + +Supports both XS and Non-XS. +Use XS for high performance, +or No-XS for ease to use (only by copying Japanese.pm). + +=item * + +Supports conversion between ZENKAKU and HANKAKU. + +=item * + +Safely handles "EMOJI" of the mobile phones (DoCoMo i-mode, ASTEL dot-i +and J-PHONE J-Sky) by mapping them on Unicode Private Use Area. + +=item * + +Supports conversion of the same image of EMOJI +between different mobile phone's standard mutually. + +=item * + +Considers Shift_JIS(SJIS) as MS-CP932. +(Shift_JIS on MS-Windows (MS-SJIS/MS-CP932) differ from +generic Shift_JIS encodings.) + +=item * + +On converting Unicode to SJIS (and EUC-JP/JIS), those encodings that cannot +be converted to SJIS (except "EMOJI") are escaped in "&#dddd;" format. +"EMOJI" on Unicode Private Use Area is going to be '?'. +When converting strings from Unicode to SJIS of mobile phones, +any characters not up to their standard is going to be '?' + +=back + +=head1 METHODS + +=over 4 + +=item $s = Unicode::Japanese->new($str [, $icode [, $encode]]) + +Creates a new instance of Unicode::Japanese. + +If arguments are specified, passes through to set method. + +=item $s->set($str [, $icode [, $encode]]) + +=over 2 + +=item $str: string + +=item $icode: character encodings, may be omitted (default = 'utf8') + +=item $encode: ASCII encoding, may be omitted. + +=back + +Set a string in the instance. +If '$icode' is omitted, string is considered as UTF-8. + +To specify a encodings, choose from the following; +'jis', 'sjis', 'euc', 'utf8', +'ucs2', 'ucs4', 'utf16', 'utf16-ge', 'utf16-le', +'utf32', 'utf32-ge', 'utf32-le', 'ascii', 'binary', +'sjis-imode', 'sjis-doti', 'sjis-jsky'. + +'&#dddd' will be converted to "EMOJI", when specified 'sjis-imode' +or 'sjis-doti'. + +For auto encoding detection, you MUST specify 'auto' +so as to call getcode() method automatically. + +For ASCII encoding, only 'base64' may be specified. +With it, the string will be decoded before storing. + +To decode binary, specify 'binary' as the encoding. + +=item $str = $s->get + +=over 2 + +=item $str: string (UTF-8) + +=back + +Gets a string with UTF-8. + +=item $code = $s->getcode($str) + +=over 2 + +=item $str: string + +=item $code: character encoding name + +=back + +Detects the character encodings of I<$str>. + +Notice: This method detects B<NOT> encoding of the string in the instance +but I<$str>. + +Character encodings are distinguished by the following algorithm: + +(In case of PurePerl) + +=over 4 + +=item 1 + +If BOM of UTF-32 is found, the encoding is utf32. + +=item 2 + +If BOM of UTF-16 is found, the encoding is utf16. + +=item 3 + +If it is in proper UTF-32BE, the encoding is utf32-be. + +=item 4 + +If it is in proper UTF-32LE, the encoding is utf32-le. + +=item 5 + +Without NON-ASCII characters, the encoding is ascii. +(control codes except escape sequences has been included in ASCII) + +=item 6 + +If it includes ISO-2022-JP(JIS) escape sequences, the encoding is jis. + +=item 7 + +If it includes "J-PHONE EMOJI", the encoding is sjis-sky. + +=item 8 + +If it is in proper EUC-JP, the encoding is euc. + +=item 9 + +If it is in proper SJIS, the encoding is sjis. + +=item 10 + +If it is in proper SJIS and "EMOJI" of i-mode, the encoding is sjis-imode. + +=item 11 + +If it is in proper SJIS and "EMOJI" of dot-i,the encoding is sjis-doti. + +=item 12 + +If it is in proper UTF-8, the encoding is utf8. + +=item 13 + +If none above is true, the encoding is unknown. + +=back + +(In case of XS) + +=over 4 + +=item 1 + +If BOM of UTF-32 is found, the encoding is utf32. + +=item 2 + +If BOM of UTF-16 is found, the encoding is utf16. + +=item 3 + +String is checked by State Transition if it is applicable +for any listed encodings below. + +ascii / euc-jp / sjis / jis / utf8 / utf32-be / utf32-le / sjis-jsky / +sjis-imode / sjis-doti + +=item 4 + +The listed order below is applied for a final determination. + +utf32-be / utf32-le / ascii / jis / euc-jp / sjis / sjis-jsky / sjis-imode / +sjis-doti / utf8 + +=item 5 + +If none above is true, the encoding is unknown. + + +=back + +Regarding the algorithm, pay attention to the following: + +=over 2 + +=item * + +UTF-8 is occasionally detected as SJIS. + +=item * + +Can NOT detect UCS2 automatically. + +=item * + +Can detect UTF-16 only when the string has BOM. + +=item * + +Can detect "EMOJI" when it is stored in binary, not in "&#dddd;" +format. (If only stored in "&#dddd;" format, getcode() will +return incorrect result. In that case, "EMOJI" will be crashed.) + +=back + +Because each of XS and PurePerl has a different algorithm, A result of +the detection would be possibly different. In case that the string is +SJIS with escape characters, it would be considered as SJIS on +PurePerl. However, it can't be detected as S-JIS on XS. This is +because by using Algorithm, the string can't be distinguished between +SJIS and SJIS-Jsky. This exclusion of escape characters on XS from +the detection is suppose to be the same for EUC-JP. + +=item $str = $s->conv($ocode, $encode) + +=over 2 + +=item $ocode: output character encoding (Choose from 'jis', 'sjis', 'euc', 'utf8', 'ucs2', 'ucs4', 'utf16', 'binary') + +=item $encode: ASCII encoding, may be omitted. + +=item $str: string + +=back + +Gets a string converted to I<$ocode>. + +For ASCII encoding, only 'base64' may be specified. With it, the string +encoded in base64 will be returned. + +=item $s->tag2bin + +Replaces the substrings "&#dddd;" in the string with the binary entity +they mean. + +=item $s->z2h + +Converts ZENKAKU to HANKAKU. + +=item $s->h2z + +Converts HANKAKU to ZENKAKU. + +=item $s->hira2kata + +Converts HIRAGANA to KATAKANA. + +=item $s->kata2hira + +Converts KATAKANA to HIRAGANA. + +=item $str = $s->jis + +$str: string (JIS) + +Gets the string converted to ISO-2022-JP(JIS). + +=item $str = $s->euc + +$str: string (EUC-JP) + +Gets the string converted to EUC-JP. + +=item $str = $s->utf8 + +$str: string (UTF-8) + +Gets the string converted to UTF-8. + +=item $str = $s->ucs2 + +$str: string (UCS2) + +Gets the string converted to UCS2. + +=item $str = $s->ucs4 + +$str: string (UCS4) + +Gets the string converted to UCS4. + +=item $str = $s->utf16 + +$str: string (UTF-16) + +Gets the string converted to UTF-16(big-endian). +BOM is not added. + +=item $str = $s->sjis + +$str: string (SJIS) + +Gets the string converted to Shift_JIS(MS-SJIS/MS-CP932). + +=item $str = $s->sjis_imode + +$str: string (SJIS/imode_EMOJI) + +Gets the string converted to SJIS for i-mode. + +=item $str = $s->sjis_doti + +$str: string (SJIS/dot-i_EMOJI) + +Gets the string converted to SJIS for dot-i. + +=item $str = $s->sjis_sky + +$str: string (SJIS/J-SKY_EMOJI) + +Gets the string converted to SJIS for j-sky. + +=item @str = $s->strcut($len) + +=over 2 + +=item $len: number of characters + +=item @str: strings + +=back + +Splits the string by length(I<$len>). + +=item $len = $s->strlen + +$len: `visual width' of the string + +Gets the length of the string. This method has been offered to +substitute for perl build-in length(). ZENKAKU characters are +assumed to have lengths of 2, regardless of the coding being +SJIS or UTF-8. + +=item $s->join_csv(@values); + +@values: data array + +Converts the array to a string in CSV format, then stores into the instance. +In the meantime, adds a newline("\n") at the end of string. + +=item @values = $s->split_csv; + +@values: data array + +Splits the string, accounting it is in CSV format. +Each newline("\n") is removed before split. + +=back + + +=head1 DESCRIPTION OF UNICODE MAPPING + +=over 2 + +=item SJIS + +Mapped as MS-CP932. Mapping table in the following URL is used. + +ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT + +If a character cannot be mapped to SJIS from Unicode, +it will be converted to &#dddd; format. + +Also, any unmapped character will be converted into "?" when converting +to SJIS for mobile phones. + +=item EUC-JP/JIS + +Converted to SJIS and then mapped to Unicode. Any non-SJIS character +in the string will not be mapped correctly. + +=item DoCoMo i-mode + +Portion of involving "EMOJI" in F800 - F9FF is maapped + to U+0FF800 - U+0FF9FF. + +=item ASTEL dot-i + +Portion of involving "EMOJI" in F000 - F4FF is mapped + to U+0FF000 - U+0FF4FF. + +=item J-PHONE J-SKY + +"J-SKY EMOJI" are mapped down as follows: "\e\$"(\x1b\x24) escape +sequences, the first byte, the second byte and "\x0f". +With sequential "EMOJI"s of identical first bytes, +it may be compressed by arranging only the second bytes. + +4500 - 47FF is mapped to U+0FFB00 - U+0FFDFF, accounting the first +and the second bytes make one EMOJI character. + +Unicode::Japanese will compress "J-SKY_EMOJI" automatically when +the first bytes of a sequence of "EMOJI" are identical. + +=back + +=head1 PurePerl mode + + use Unicode::Japanese qw(PurePerl); + +If module was loaded with 'PurePerl' keyword, +it works on Non-XS mode. + +=head1 BUGS + +=over 2 + +=item * + +EUC-JP, JIS strings cannot be converted correctly when they include +non-SJIS characters because they are converted to SJIS before +being converted to UTF-8. + +=item * + +Some characters of CP932 not in standard Shift_JIS +(ex; not in Joyo Kanji) will not be detected and converted. + +When string include such non-standard Shift_JIS, +they will not detected as SJIS. +Also, getcode() and all convert method will not work correctly. + +=item * + +When using XS, character encoding detection of EUC-JP and +SJIS(included all EMOJI) strings when they include "\e" will +fail. Also, getcode() and all convert method will not work. + +=item * + +The Japanese.pm file will collapse if sent via ASCII mode of FTP, +as it has a trailing binary data. + +=back + +=head1 AUTHOR INFORMATION + +Copyright 2001-2002 +SANO Taku (SAWATARI Mikage) and YAMASHINA Hio. +All right reserved. + +This library is free software; you can redistribute it +and/or modify it under the same terms as Perl itself. + +Bug reports and comments to: mikage@cpan.org. +Thank you. + +=head1 CREDITS + +Thanks very much to: + +NAKAYAMA Nao + +SUGIURA Tatsuki & Debian JP Project + +=cut + + + +__DATA__ + + + +%_h2zNum = ( + "0" => "\xef\xbc\x90", "1" => "\xef\xbc\x91", + "2" => "\xef\xbc\x92", "3" => "\xef\xbc\x93", + "4" => "\xef\xbc\x94", "5" => "\xef\xbc\x95", + "6" => "\xef\xbc\x96", "7" => "\xef\xbc\x97", + "8" => "\xef\xbc\x98", "9" => "\xef\xbc\x99", + +); + + + +%_z2hNum = ( + "\xef\xbc\x90" => "0", "\xef\xbc\x91" => "1", + "\xef\xbc\x92" => "2", "\xef\xbc\x93" => "3", + "\xef\xbc\x94" => "4", "\xef\xbc\x95" => "5", + "\xef\xbc\x96" => "6", "\xef\xbc\x97" => "7", + "\xef\xbc\x98" => "8", "\xef\xbc\x99" => "9", + +); + + + +%_h2zAlpha = ( + "A" => "\xef\xbc\xa1", "B" => "\xef\xbc\xa2", + "C" => "\xef\xbc\xa3", "D" => "\xef\xbc\xa4", + "E" => "\xef\xbc\xa5", "F" => "\xef\xbc\xa6", + "G" => "\xef\xbc\xa7", "H" => "\xef\xbc\xa8", + "I" => "\xef\xbc\xa9", "J" => "\xef\xbc\xaa", + "K" => "\xef\xbc\xab", "L" => "\xef\xbc\xac", + "M" => "\xef\xbc\xad", "N" => "\xef\xbc\xae", + "O" => "\xef\xbc\xaf", "P" => "\xef\xbc\xb0", + "Q" => "\xef\xbc\xb1", "R" => "\xef\xbc\xb2", + "S" => "\xef\xbc\xb3", "T" => "\xef\xbc\xb4", + "U" => "\xef\xbc\xb5", "V" => "\xef\xbc\xb6", + "W" => "\xef\xbc\xb7", "X" => "\xef\xbc\xb8", + "Y" => "\xef\xbc\xb9", "Z" => "\xef\xbc\xba", + "a" => "\xef\xbd\x81", "b" => "\xef\xbd\x82", + "c" => "\xef\xbd\x83", "d" => "\xef\xbd\x84", + "e" => "\xef\xbd\x85", "f" => "\xef\xbd\x86", + "g" => "\xef\xbd\x87", "h" => "\xef\xbd\x88", + "i" => "\xef\xbd\x89", "j" => "\xef\xbd\x8a", + "k" => "\xef\xbd\x8b", "l" => "\xef\xbd\x8c", + "m" => "\xef\xbd\x8d", "n" => "\xef\xbd\x8e", + "o" => "\xef\xbd\x8f", "p" => "\xef\xbd\x90", + "q" => "\xef\xbd\x91", "r" => "\xef\xbd\x92", + "s" => "\xef\xbd\x93", "t" => "\xef\xbd\x94", + "u" => "\xef\xbd\x95", "v" => "\xef\xbd\x96", + "w" => "\xef\xbd\x97", "x" => "\xef\xbd\x98", + "y" => "\xef\xbd\x99", "z" => "\xef\xbd\x9a", + +); + + + +%_z2hAlpha = ( + "\xef\xbc\xa1" => "A", "\xef\xbc\xa2" => "B", + "\xef\xbc\xa3" => "C", "\xef\xbc\xa4" => "D", + "\xef\xbc\xa5" => "E", "\xef\xbc\xa6" => "F", + "\xef\xbc\xa7" => "G", "\xef\xbc\xa8" => "H", + "\xef\xbc\xa9" => "I", "\xef\xbc\xaa" => "J", + "\xef\xbc\xab" => "K", "\xef\xbc\xac" => "L", + "\xef\xbc\xad" => "M", "\xef\xbc\xae" => "N", + "\xef\xbc\xaf" => "O", "\xef\xbc\xb0" => "P", + "\xef\xbc\xb1" => "Q", "\xef\xbc\xb2" => "R", + "\xef\xbc\xb3" => "S", "\xef\xbc\xb4" => "T", + "\xef\xbc\xb5" => "U", "\xef\xbc\xb6" => "V", + "\xef\xbc\xb7" => "W", "\xef\xbc\xb8" => "X", + "\xef\xbc\xb9" => "Y", "\xef\xbc\xba" => "Z", + "\xef\xbd\x81" => "a", "\xef\xbd\x82" => "b", + "\xef\xbd\x83" => "c", "\xef\xbd\x84" => "d", + "\xef\xbd\x85" => "e", "\xef\xbd\x86" => "f", + "\xef\xbd\x87" => "g", "\xef\xbd\x88" => "h", + "\xef\xbd\x89" => "i", "\xef\xbd\x8a" => "j", + "\xef\xbd\x8b" => "k", "\xef\xbd\x8c" => "l", + "\xef\xbd\x8d" => "m", "\xef\xbd\x8e" => "n", + "\xef\xbd\x8f" => "o", "\xef\xbd\x90" => "p", + "\xef\xbd\x91" => "q", "\xef\xbd\x92" => "r", + "\xef\xbd\x93" => "s", "\xef\xbd\x94" => "t", + "\xef\xbd\x95" => "u", "\xef\xbd\x96" => "v", + "\xef\xbd\x97" => "w", "\xef\xbd\x98" => "x", + "\xef\xbd\x99" => "y", "\xef\xbd\x9a" => "z", + +); + + + +%_h2zSym = ( + "\x20" => "\xe3\x80\x80", "\x21" => "\xef\xbc\x81", + "\x22" => "\xe2\x80\x9d", "\x23" => "\xef\xbc\x83", + "\x24" => "\xef\xbc\x84", "\x25" => "\xef\xbc\x85", + "\x26" => "\xef\xbc\x86", "\x27" => "\xef\xbf\xa5", + "\x28" => "\xef\xbc\x88", "\x29" => "\xef\xbc\x89", + "\x2a" => "\xef\xbc\x8a", "\x2b" => "\xef\xbc\x8b", + "\x2c" => "\xef\xbc\x8c", "\x2d" => "\xe2\x88\x92", + "\x2e" => "\xef\xbc\x8e", "\x2f" => "\xef\xbc\x8f", + "\x3a" => "\xef\xbc\x9a", "\x3b" => "\xef\xbc\x9b", + "\x3c" => "\xef\xbc\x9c", "\x3d" => "\xef\xbc\x9d", + "\x3e" => "\xef\xbc\x9e", "\x3f" => "\xef\xbc\x9f", + "\x40" => "\xef\xbc\xa0", "\x5b" => "\xef\xbc\xbb", + "\x5c" => "\xef\xbf\xa5", "\x5d" => "\xef\xbc\xbd", + "\x5e" => "\xef\xbc\xbe", "\x60" => "\xef\xbd\x80", + "\x7b" => "\xef\xbd\x9b", "\x7c" => "\xef\xbd\x9c", + "\x7d" => "\xef\xbd\x9d", "\x7e" => "\xe3\x80\x9c", + +); + + + +%_z2hSym = ( + "\xe3\x80\x80" => "\x20", "\xef\xbc\x8c" => "\x2c", + "\xef\xbc\x8e" => "\x2e", "\xef\xbc\x9a" => "\x3a", + "\xef\xbc\x9b" => "\x3b", "\xef\xbc\x9f" => "\x3f", + "\xef\xbc\x81" => "\x21", "\xef\xbd\x80" => "\x60", + "\xef\xbc\xbe" => "\x5e", "\xef\xbc\x8f" => "\x2f", + "\xe3\x80\x9c" => "\x7e", "\xef\xbd\x9c" => "\x7c", + "\xe2\x80\x9d" => "\x22", "\xef\xbc\x88" => "\x28", + "\xef\xbc\x89" => "\x29", "\xef\xbc\xbb" => "\x5b", + "\xef\xbc\xbd" => "\x5d", "\xef\xbd\x9b" => "\x7b", + "\xef\xbd\x9d" => "\x7d", "\xef\xbc\x8b" => "\x2b", + "\xe2\x88\x92" => "\x2d", "\xef\xbc\x9d" => "\x3d", + "\xef\xbc\x9c" => "\x3c", "\xef\xbc\x9e" => "\x3e", + "\xef\xbf\xa5" => "\x27", "\xef\xbc\x84" => "\x24", + "\xef\xbc\x85" => "\x25", "\xef\xbc\x83" => "\x23", + "\xef\xbc\x86" => "\x26", "\xef\xbc\x8a" => "\x2a", + "\xef\xbc\xa0" => "\x40", +); + + + +%_h2zKanaK = ( + "\xef\xbd\xa1" => "\xe3\x80\x82", "\xef\xbd\xa2" => "\xe3\x80\x8c", + "\xef\xbd\xa3" => "\xe3\x80\x8d", "\xef\xbd\xa4" => "\xe3\x80\x81", + "\xef\xbd\xa5" => "\xe3\x83\xbb", "\xef\xbd\xa6" => "\xe3\x83\xb2", + "\xef\xbd\xa7" => "\xe3\x82\xa1", "\xef\xbd\xa8" => "\xe3\x82\xa3", + "\xef\xbd\xa9" => "\xe3\x82\xa5", "\xef\xbd\xaa" => "\xe3\x82\xa7", + "\xef\xbd\xab" => "\xe3\x82\xa9", "\xef\xbd\xac" => "\xe3\x83\xa3", + "\xef\xbd\xad" => "\xe3\x83\xa5", "\xef\xbd\xae" => "\xe3\x83\xa7", + "\xef\xbd\xaf" => "\xe3\x83\x83", "\xef\xbd\xb0" => "\xe3\x83\xbc", + "\xef\xbd\xb1" => "\xe3\x82\xa2", "\xef\xbd\xb2" => "\xe3\x82\xa4", + "\xef\xbd\xb3" => "\xe3\x82\xa6", "\xef\xbd\xb4" => "\xe3\x82\xa8", + "\xef\xbd\xb5" => "\xe3\x82\xaa", "\xef\xbd\xb6" => "\xe3\x82\xab", + "\xef\xbd\xb7" => "\xe3\x82\xad", "\xef\xbd\xb8" => "\xe3\x82\xaf", + "\xef\xbd\xb9" => "\xe3\x82\xb1", "\xef\xbd\xba" => "\xe3\x82\xb3", + "\xef\xbd\xbb" => "\xe3\x82\xb5", "\xef\xbd\xbc" => "\xe3\x82\xb7", + "\xef\xbd\xbd" => "\xe3\x82\xb9", "\xef\xbd\xbe" => "\xe3\x82\xbb", + "\xef\xbd\xbf" => "\xe3\x82\xbd", "\xef\xbe\x80" => "\xe3\x82\xbf", + "\xef\xbe\x81" => "\xe3\x83\x81", "\xef\xbe\x82" => "\xe3\x83\x84", + "\xef\xbe\x83" => "\xe3\x83\x86", "\xef\xbe\x84" => "\xe3\x83\x88", + "\xef\xbe\x85" => "\xe3\x83\x8a", "\xef\xbe\x86" => "\xe3\x83\x8b", + "\xef\xbe\x87" => "\xe3\x83\x8c", "\xef\xbe\x88" => "\xe3\x83\x8d", + "\xef\xbe\x89" => "\xe3\x83\x8e", "\xef\xbe\x8a" => "\xe3\x83\x8f", + "\xef\xbe\x8b" => "\xe3\x83\x92", "\xef\xbe\x8c" => "\xe3\x83\x95", + "\xef\xbe\x8d" => "\xe3\x83\x98", "\xef\xbe\x8e" => "\xe3\x83\x9b", + "\xef\xbe\x8f" => "\xe3\x83\x9e", "\xef\xbe\x90" => "\xe3\x83\x9f", + "\xef\xbe\x91" => "\xe3\x83\xa0", "\xef\xbe\x92" => "\xe3\x83\xa1", + "\xef\xbe\x93" => "\xe3\x83\xa2", "\xef\xbe\x94" => "\xe3\x83\xa4", + "\xef\xbe\x95" => "\xe3\x83\xa6", "\xef\xbe\x96" => "\xe3\x83\xa8", + "\xef\xbe\x97" => "\xe3\x83\xa9", "\xef\xbe\x98" => "\xe3\x83\xaa", + "\xef\xbe\x99" => "\xe3\x83\xab", "\xef\xbe\x9a" => "\xe3\x83\xac", + "\xef\xbe\x9b" => "\xe3\x83\xad", "\xef\xbe\x9c" => "\xe3\x83\xaf", + "\xef\xbe\x9d" => "\xe3\x83\xb3", "\xef\xbe\x9e" => "\xe3\x82\x9b", + "\xef\xbe\x9f" => "\xe3\x82\x9c", +); + + + +%_z2hKanaK = ( + "\xe3\x80\x81" => "\xef\xbd\xa4", "\xe3\x80\x82" => "\xef\xbd\xa1", + "\xe3\x83\xbb" => "\xef\xbd\xa5", "\xe3\x82\x9b" => "\xef\xbe\x9e", + "\xe3\x82\x9c" => "\xef\xbe\x9f", "\xe3\x83\xbc" => "\xef\xbd\xb0", + "\xe3\x80\x8c" => "\xef\xbd\xa2", "\xe3\x80\x8d" => "\xef\xbd\xa3", + "\xe3\x82\xa1" => "\xef\xbd\xa7", "\xe3\x82\xa2" => "\xef\xbd\xb1", + "\xe3\x82\xa3" => "\xef\xbd\xa8", "\xe3\x82\xa4" => "\xef\xbd\xb2", + "\xe3\x82\xa5" => "\xef\xbd\xa9", "\xe3\x82\xa6" => "\xef\xbd\xb3", + "\xe3\x82\xa7" => "\xef\xbd\xaa", "\xe3\x82\xa8" => "\xef\xbd\xb4", + "\xe3\x82\xa9" => "\xef\xbd\xab", "\xe3\x82\xaa" => "\xef\xbd\xb5", + "\xe3\x82\xab" => "\xef\xbd\xb6", "\xe3\x82\xad" => "\xef\xbd\xb7", + "\xe3\x82\xaf" => "\xef\xbd\xb8", "\xe3\x82\xb1" => "\xef\xbd\xb9", + "\xe3\x82\xb3" => "\xef\xbd\xba", "\xe3\x82\xb5" => "\xef\xbd\xbb", + "\xe3\x82\xb7" => "\xef\xbd\xbc", "\xe3\x82\xb9" => "\xef\xbd\xbd", + "\xe3\x82\xbb" => "\xef\xbd\xbe", "\xe3\x82\xbd" => "\xef\xbd\xbf", + "\xe3\x82\xbf" => "\xef\xbe\x80", "\xe3\x83\x81" => "\xef\xbe\x81", + "\xe3\x83\x83" => "\xef\xbd\xaf", "\xe3\x83\x84" => "\xef\xbe\x82", + "\xe3\x83\x86" => "\xef\xbe\x83", "\xe3\x83\x88" => "\xef\xbe\x84", + "\xe3\x83\x8a" => "\xef\xbe\x85", "\xe3\x83\x8b" => "\xef\xbe\x86", + "\xe3\x83\x8c" => "\xef\xbe\x87", "\xe3\x83\x8d" => "\xef\xbe\x88", + "\xe3\x83\x8e" => "\xef\xbe\x89", "\xe3\x83\x8f" => "\xef\xbe\x8a", + "\xe3\x83\x92" => "\xef\xbe\x8b", "\xe3\x83\x95" => "\xef\xbe\x8c", + "\xe3\x83\x98" => "\xef\xbe\x8d", "\xe3\x83\x9b" => "\xef\xbe\x8e", + "\xe3\x83\x9e" => "\xef\xbe\x8f", "\xe3\x83\x9f" => "\xef\xbe\x90", + "\xe3\x83\xa0" => "\xef\xbe\x91", "\xe3\x83\xa1" => "\xef\xbe\x92", + "\xe3\x83\xa2" => "\xef\xbe\x93", "\xe3\x83\xa3" => "\xef\xbd\xac", + "\xe3\x83\xa4" => "\xef\xbe\x94", "\xe3\x83\xa5" => "\xef\xbd\xad", + "\xe3\x83\xa6" => "\xef\xbe\x95", "\xe3\x83\xa7" => "\xef\xbd\xae", + "\xe3\x83\xa8" => "\xef\xbe\x96", "\xe3\x83\xa9" => "\xef\xbe\x97", + "\xe3\x83\xaa" => "\xef\xbe\x98", "\xe3\x83\xab" => "\xef\xbe\x99", + "\xe3\x83\xac" => "\xef\xbe\x9a", "\xe3\x83\xad" => "\xef\xbe\x9b", + "\xe3\x83\xaf" => "\xef\xbe\x9c", "\xe3\x83\xb2" => "\xef\xbd\xa6", + "\xe3\x83\xb3" => "\xef\xbe\x9d", +); + + + +%_h2zKanaD = ( + "\xef\xbd\xb3\xef\xbe\x9e" => "\xe3\x83\xb4", "\xef\xbd\xb6\xef\xbe\x9e" => "\xe3\x82\xac", + "\xef\xbd\xb7\xef\xbe\x9e" => "\xe3\x82\xae", "\xef\xbd\xb8\xef\xbe\x9e" => "\xe3\x82\xb0", + "\xef\xbd\xb9\xef\xbe\x9e" => "\xe3\x82\xb2", "\xef\xbd\xba\xef\xbe\x9e" => "\xe3\x82\xb4", + "\xef\xbd\xbb\xef\xbe\x9e" => "\xe3\x82\xb6", "\xef\xbd\xbc\xef\xbe\x9e" => "\xe3\x82\xb8", + "\xef\xbd\xbd\xef\xbe\x9e" => "\xe3\x82\xba", "\xef\xbd\xbe\xef\xbe\x9e" => "\xe3\x82\xbc", + "\xef\xbd\xbf\xef\xbe\x9e" => "\xe3\x82\xbe", "\xef\xbe\x80\xef\xbe\x9e" => "\xe3\x83\x80", + "\xef\xbe\x81\xef\xbe\x9e" => "\xe3\x83\x82", "\xef\xbe\x82\xef\xbe\x9e" => "\xe3\x83\x85", + "\xef\xbe\x83\xef\xbe\x9e" => "\xe3\x83\x87", "\xef\xbe\x84\xef\xbe\x9e" => "\xe3\x83\x89", + "\xef\xbe\x8a\xef\xbe\x9e" => "\xe3\x83\x90", "\xef\xbe\x8a\xef\xbe\x9f" => "\xe3\x83\x91", + "\xef\xbe\x8b\xef\xbe\x9e" => "\xe3\x83\x93", "\xef\xbe\x8b\xef\xbe\x9f" => "\xe3\x83\x94", + "\xef\xbe\x8c\xef\xbe\x9e" => "\xe3\x83\x96", "\xef\xbe\x8c\xef\xbe\x9f" => "\xe3\x83\x97", + "\xef\xbe\x8d\xef\xbe\x9e" => "\xe3\x83\x99", "\xef\xbe\x8d\xef\xbe\x9f" => "\xe3\x83\x9a", + "\xef\xbe\x8e\xef\xbe\x9e" => "\xe3\x83\x9c", "\xef\xbe\x8e\xef\xbe\x9f" => "\xe3\x83\x9d", + +); + + + +%_z2hKanaD = ( + "\xe3\x82\xac" => "\xef\xbd\xb6\xef\xbe\x9e", "\xe3\x82\xae" => "\xef\xbd\xb7\xef\xbe\x9e", + "\xe3\x82\xb0" => "\xef\xbd\xb8\xef\xbe\x9e", "\xe3\x82\xb2" => "\xef\xbd\xb9\xef\xbe\x9e", + "\xe3\x82\xb4" => "\xef\xbd\xba\xef\xbe\x9e", "\xe3\x82\xb6" => "\xef\xbd\xbb\xef\xbe\x9e", + "\xe3\x82\xb8" => "\xef\xbd\xbc\xef\xbe\x9e", "\xe3\x82\xba" => "\xef\xbd\xbd\xef\xbe\x9e", + "\xe3\x82\xbc" => "\xef\xbd\xbe\xef\xbe\x9e", "\xe3\x82\xbe" => "\xef\xbd\xbf\xef\xbe\x9e", + "\xe3\x83\x80" => "\xef\xbe\x80\xef\xbe\x9e", "\xe3\x83\x82" => "\xef\xbe\x81\xef\xbe\x9e", + "\xe3\x83\x85" => "\xef\xbe\x82\xef\xbe\x9e", "\xe3\x83\x87" => "\xef\xbe\x83\xef\xbe\x9e", + "\xe3\x83\x89" => "\xef\xbe\x84\xef\xbe\x9e", "\xe3\x83\x90" => "\xef\xbe\x8a\xef\xbe\x9e", + "\xe3\x83\x91" => "\xef\xbe\x8a\xef\xbe\x9f", "\xe3\x83\x93" => "\xef\xbe\x8b\xef\xbe\x9e", + "\xe3\x83\x94" => "\xef\xbe\x8b\xef\xbe\x9f", "\xe3\x83\x96" => "\xef\xbe\x8c\xef\xbe\x9e", + "\xe3\x83\x97" => "\xef\xbe\x8c\xef\xbe\x9f", "\xe3\x83\x99" => "\xef\xbe\x8d\xef\xbe\x9e", + "\xe3\x83\x9a" => "\xef\xbe\x8d\xef\xbe\x9f", "\xe3\x83\x9c" => "\xef\xbe\x8e\xef\xbe\x9e", + "\xe3\x83\x9d" => "\xef\xbe\x8e\xef\xbe\x9f", "\xe3\x83\xb4" => "\xef\xbd\xb3\xef\xbe\x9e", + +); + + + +%_hira2kata = ( + "\xe3\x81\x81" => "\xe3\x82\xa1", "\xe3\x81\x82" => "\xe3\x82\xa2", + "\xe3\x81\x83" => "\xe3\x82\xa3", "\xe3\x81\x84" => "\xe3\x82\xa4", + "\xe3\x81\x85" => "\xe3\x82\xa5", "\xe3\x81\x86" => "\xe3\x82\xa6", + "\xe3\x81\x87" => "\xe3\x82\xa7", "\xe3\x81\x88" => "\xe3\x82\xa8", + "\xe3\x81\x89" => "\xe3\x82\xa9", "\xe3\x81\x8a" => "\xe3\x82\xaa", + "\xe3\x81\x8b" => "\xe3\x82\xab", "\xe3\x81\x8c" => "\xe3\x82\xac", + "\xe3\x81\x8d" => "\xe3\x82\xad", "\xe3\x81\x8e" => "\xe3\x82\xae", + "\xe3\x81\x8f" => "\xe3\x82\xaf", "\xe3\x81\x90" => "\xe3\x82\xb0", + "\xe3\x81\x91" => "\xe3\x82\xb1", "\xe3\x81\x92" => "\xe3\x82\xb2", + "\xe3\x81\x93" => "\xe3\x82\xb3", "\xe3\x81\x94" => "\xe3\x82\xb4", + "\xe3\x81\x95" => "\xe3\x82\xb5", "\xe3\x81\x96" => "\xe3\x82\xb6", + "\xe3\x81\x97" => "\xe3\x82\xb7", "\xe3\x81\x98" => "\xe3\x82\xb8", + "\xe3\x81\x99" => "\xe3\x82\xb9", "\xe3\x81\x9a" => "\xe3\x82\xba", + "\xe3\x81\x9b" => "\xe3\x82\xbb", "\xe3\x81\x9c" => "\xe3\x82\xbc", + "\xe3\x81\x9d" => "\xe3\x82\xbd", "\xe3\x81\x9e" => "\xe3\x82\xbe", + "\xe3\x81\x9f" => "\xe3\x82\xbf", "\xe3\x81\xa0" => "\xe3\x83\x80", + "\xe3\x81\xa1" => "\xe3\x83\x81", "\xe3\x81\xa2" => "\xe3\x83\x82", + "\xe3\x81\xa3" => "\xe3\x83\x83", "\xe3\x81\xa4" => "\xe3\x83\x84", + "\xe3\x81\xa5" => "\xe3\x83\x85", "\xe3\x81\xa6" => "\xe3\x83\x86", + "\xe3\x81\xa7" => "\xe3\x83\x87", "\xe3\x81\xa8" => "\xe3\x83\x88", + "\xe3\x81\xa9" => "\xe3\x83\x89", "\xe3\x81\xaa" => "\xe3\x83\x8a", + "\xe3\x81\xab" => "\xe3\x83\x8b", "\xe3\x81\xac" => "\xe3\x83\x8c", + "\xe3\x81\xad" => "\xe3\x83\x8d", "\xe3\x81\xae" => "\xe3\x83\x8e", + "\xe3\x81\xaf" => "\xe3\x83\x8f", "\xe3\x81\xb0" => "\xe3\x83\x90", + "\xe3\x81\xb1" => "\xe3\x83\x91", "\xe3\x81\xb2" => "\xe3\x83\x92", + "\xe3\x81\xb3" => "\xe3\x83\x93", "\xe3\x81\xb4" => "\xe3\x83\x94", + "\xe3\x81\xb5" => "\xe3\x83\x95", "\xe3\x81\xb6" => "\xe3\x83\x96", + "\xe3\x81\xb7" => "\xe3\x83\x97", "\xe3\x81\xb8" => "\xe3\x83\x98", + "\xe3\x81\xb9" => "\xe3\x83\x99", "\xe3\x81\xba" => "\xe3\x83\x9a", + "\xe3\x81\xbb" => "\xe3\x83\x9b", "\xe3\x81\xbc" => "\xe3\x83\x9c", + "\xe3\x81\xbd" => "\xe3\x83\x9d", "\xe3\x81\xbe" => "\xe3\x83\x9e", + "\xe3\x81\xbf" => "\xe3\x83\x9f", "\xe3\x82\x80" => "\xe3\x83\xa0", + "\xe3\x82\x81" => "\xe3\x83\xa1", "\xe3\x82\x82" => "\xe3\x83\xa2", + "\xe3\x82\x83" => "\xe3\x83\xa3", "\xe3\x82\x84" => "\xe3\x83\xa4", + "\xe3\x82\x85" => "\xe3\x83\xa5", "\xe3\x82\x86" => "\xe3\x83\xa6", + "\xe3\x82\x87" => "\xe3\x83\xa7", "\xe3\x82\x88" => "\xe3\x83\xa8", + "\xe3\x82\x89" => "\xe3\x83\xa9", "\xe3\x82\x8a" => "\xe3\x83\xaa", + "\xe3\x82\x8b" => "\xe3\x83\xab", "\xe3\x82\x8c" => "\xe3\x83\xac", + "\xe3\x82\x8d" => "\xe3\x83\xad", "\xe3\x82\x8e" => "\xe3\x83\xae", + "\xe3\x82\x8f" => "\xe3\x83\xaf", "\xe3\x82\x90" => "\xe3\x83\xb0", + "\xe3\x82\x91" => "\xe3\x83\xb1", "\xe3\x82\x92" => "\xe3\x83\xb2", + "\xe3\x82\x93" => "\xe3\x83\xb3", +); + + + +%_kata2hira = ( + "\xe3\x82\xa1" => "\xe3\x81\x81", "\xe3\x82\xa2" => "\xe3\x81\x82", + "\xe3\x82\xa3" => "\xe3\x81\x83", "\xe3\x82\xa4" => "\xe3\x81\x84", + "\xe3\x82\xa5" => "\xe3\x81\x85", "\xe3\x82\xa6" => "\xe3\x81\x86", + "\xe3\x82\xa7" => "\xe3\x81\x87", "\xe3\x82\xa8" => "\xe3\x81\x88", + "\xe3\x82\xa9" => "\xe3\x81\x89", "\xe3\x82\xaa" => "\xe3\x81\x8a", + "\xe3\x82\xab" => "\xe3\x81\x8b", "\xe3\x82\xac" => "\xe3\x81\x8c", + "\xe3\x82\xad" => "\xe3\x81\x8d", "\xe3\x82\xae" => "\xe3\x81\x8e", + "\xe3\x82\xaf" => "\xe3\x81\x8f", "\xe3\x82\xb0" => "\xe3\x81\x90", + "\xe3\x82\xb1" => "\xe3\x81\x91", "\xe3\x82\xb2" => "\xe3\x81\x92", + "\xe3\x82\xb3" => "\xe3\x81\x93", "\xe3\x82\xb4" => "\xe3\x81\x94", + "\xe3\x82\xb5" => "\xe3\x81\x95", "\xe3\x82\xb6" => "\xe3\x81\x96", + "\xe3\x82\xb7" => "\xe3\x81\x97", "\xe3\x82\xb8" => "\xe3\x81\x98", + "\xe3\x82\xb9" => "\xe3\x81\x99", "\xe3\x82\xba" => "\xe3\x81\x9a", + "\xe3\x82\xbb" => "\xe3\x81\x9b", "\xe3\x82\xbc" => "\xe3\x81\x9c", + "\xe3\x82\xbd" => "\xe3\x81\x9d", "\xe3\x82\xbe" => "\xe3\x81\x9e", + "\xe3\x82\xbf" => "\xe3\x81\x9f", "\xe3\x83\x80" => "\xe3\x81\xa0", + "\xe3\x83\x81" => "\xe3\x81\xa1", "\xe3\x83\x82" => "\xe3\x81\xa2", + "\xe3\x83\x83" => "\xe3\x81\xa3", "\xe3\x83\x84" => "\xe3\x81\xa4", + "\xe3\x83\x85" => "\xe3\x81\xa5", "\xe3\x83\x86" => "\xe3\x81\xa6", + "\xe3\x83\x87" => "\xe3\x81\xa7", "\xe3\x83\x88" => "\xe3\x81\xa8", + "\xe3\x83\x89" => "\xe3\x81\xa9", "\xe3\x83\x8a" => "\xe3\x81\xaa", + "\xe3\x83\x8b" => "\xe3\x81\xab", "\xe3\x83\x8c" => "\xe3\x81\xac", + "\xe3\x83\x8d" => "\xe3\x81\xad", "\xe3\x83\x8e" => "\xe3\x81\xae", + "\xe3\x83\x8f" => "\xe3\x81\xaf", "\xe3\x83\x90" => "\xe3\x81\xb0", + "\xe3\x83\x91" => "\xe3\x81\xb1", "\xe3\x83\x92" => "\xe3\x81\xb2", + "\xe3\x83\x93" => "\xe3\x81\xb3", "\xe3\x83\x94" => "\xe3\x81\xb4", + "\xe3\x83\x95" => "\xe3\x81\xb5", "\xe3\x83\x96" => "\xe3\x81\xb6", + "\xe3\x83\x97" => "\xe3\x81\xb7", "\xe3\x83\x98" => "\xe3\x81\xb8", + "\xe3\x83\x99" => "\xe3\x81\xb9", "\xe3\x83\x9a" => "\xe3\x81\xba", + "\xe3\x83\x9b" => "\xe3\x81\xbb", "\xe3\x83\x9c" => "\xe3\x81\xbc", + "\xe3\x83\x9d" => "\xe3\x81\xbd", "\xe3\x83\x9e" => "\xe3\x81\xbe", + "\xe3\x83\x9f" => "\xe3\x81\xbf", "\xe3\x83\xa0" => "\xe3\x82\x80", + "\xe3\x83\xa1" => "\xe3\x82\x81", "\xe3\x83\xa2" => "\xe3\x82\x82", + "\xe3\x83\xa3" => "\xe3\x82\x83", "\xe3\x83\xa4" => "\xe3\x82\x84", + "\xe3\x83\xa5" => "\xe3\x82\x85", "\xe3\x83\xa6" => "\xe3\x82\x86", + "\xe3\x83\xa7" => "\xe3\x82\x87", "\xe3\x83\xa8" => "\xe3\x82\x88", + "\xe3\x83\xa9" => "\xe3\x82\x89", "\xe3\x83\xaa" => "\xe3\x82\x8a", + "\xe3\x83\xab" => "\xe3\x82\x8b", "\xe3\x83\xac" => "\xe3\x82\x8c", + "\xe3\x83\xad" => "\xe3\x82\x8d", "\xe3\x83\xae" => "\xe3\x82\x8e", + "\xe3\x83\xaf" => "\xe3\x82\x8f", "\xe3\x83\xb0" => "\xe3\x82\x90", + "\xe3\x83\xb1" => "\xe3\x82\x91", "\xe3\x83\xb2" => "\xe3\x82\x92", + "\xe3\x83\xb3" => "\xe3\x82\x93", +); + + +} +sub h2zNum { + my $this = shift; + + if(!defined(%_h2zNum)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(0|1|2|3|4|5|6|7|8|9)/$_h2zNum{$1}/eg; + + $this; +} +sub euc +{ + my $this = shift; + $this->_s2e($this->sjis); +} +sub z2hKana +{ + my $this = shift; + + $this->z2hKanaD; + $this->z2hKanaK; + + $this; +} +sub splitCsv { + my $this = shift; + my $text = $this->{str}; + my @field; + + chomp($text); + + while ($text =~ m/"([^"\\]*(?:(?:\\.|\"\")[^"\\]*)*)",?|([^,]+),?|,/g) { + my $field = defined($1) ? $1 : (defined($2) ? $2 : ''); + $field =~ s/["\\]"/"/g; + push(@field, $field); + } + push(@field, '') if($text =~ m/,$/); + + \@field; + +} +sub strlen { + my $this = shift; + + my $ch_re = '[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}'; + my $length = 0; + + foreach my $c(split(/($ch_re)/,$this->{str})) { + next if(length($c) == 0); + $length += ((length($c) >= 3) ? 2 : 1); + } + + return $length; +} +sub join_csv { + my $this = shift; + + $this->joinCsv(@_); +} +sub utf16 +{ + my $this = shift; + $this->_utf8_utf16($this->{str}); +} +sub _utf16_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + my $sa; + foreach my $uc (unpack("n*", $str)) + { + ($uc >= 0xd800 and $uc <= 0xdbff and $sa = $uc and next); + + ($uc >= 0xdc00 and $uc <= 0xdfff and ($uc = ((($sa - 0xd800) << 10)|($uc - 0xdc00))+0x10000)); + + $result .= $U2T[$uc] ? $U2T[$uc] : + ($U2T[$uc] = ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x10000) ? chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xF0 | ($uc >> 18)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F))); + } + + $result; +} +sub _u2s { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : ( + $U2S{$1} + or ($U2S{$1} + = ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + # UTF-3バイト(U+0x80-U+07FF)からsjis-1バイトへのマッピングはないので\0を削除は必要はない + ($c eq "\0\0") ? '&#' . $ch . ';' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '&#' . $ch . ';' + ) + ), + ($c eq "\0\0") ? '&#' . $ch . ';' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + '?' + : '&#' . $ch . ';' + ) + ) : + (length($1) == 5) ? (($c1,$c2,$c3,$c4,$c5) = unpack("C5", $1), + $ch = (($c1 & 0x03) << 24)|(($c2 & 0x3F) << 18)| + (($c3 & 0x3f) << 12)|(($c4 & 0x3f) << 6)| + ($c5 & 0x3F), + '&#' . $ch . ';' + ) : + ( + ($c1,$c2,$c3,$c4,$c5,$c6) = unpack("C6", $1), + $ch = (($c1 & 0x03) << 30)|(($c2 & 0x3F) << 24)| + (($c3 & 0x3f) << 18)|(($c4 & 0x3f) << 12)| + (($c5 & 0x3f) << 6)|($c6 & 0x3F), + '&#' . $ch . ';' + ) + ) + ) + ) + /eg; + $str; + +} +sub _j2s2 { + my $this = shift; + my $esc = shift; + my $str = shift; + + if($esc eq $RE{JIS_0212}) + { + $str =~ s/../$CHARCODE{UNDEF_SJIS}/g; + } + elsif($esc !~ m/^$RE{JIS_ASC}/) + { + $str =~ tr/\x21-\x7e/\xa1-\xfe/; + if($esc =~ m/^$RE{JIS_0208}/) + { + $str =~ s/($RE{EUC_C})/ + $J2S[unpack('n', $1)] or $this->_j2s3($1) + /geo; + } + } + + $str; +} +sub z2hKanaD { + my $this = shift; + + if(!defined(%_z2hKanaD)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x82\xac|\xe3\x82\xae|\xe3\x82\xb0|\xe3\x82\xb2|\xe3\x82\xb4|\xe3\x82\xb6|\xe3\x82\xb8|\xe3\x82\xba|\xe3\x82\xbc|\xe3\x82\xbe|\xe3\x83\x80|\xe3\x83\x82|\xe3\x83\x85|\xe3\x83\x87|\xe3\x83\x89|\xe3\x83\x90|\xe3\x83\x91|\xe3\x83\x93|\xe3\x83\x94|\xe3\x83\x96|\xe3\x83\x97|\xe3\x83\x99|\xe3\x83\x9a|\xe3\x83\x9c|\xe3\x83\x9d|\xe3\x83\xb4)/$_z2hKanaD{$1}/eg; + + $this; +} +sub _j2s3 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if ($c1 % 2) + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x31 : 0x71); + $c2 -= 0x60 + ($c2 < 0xe0); + } + else + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x30 : 0x70); + $c2 -= 2; + } + + $J2S[unpack('n', $c)] = pack('CC', $c1, $c2); +} +sub joinCsv { + my $this = shift; + my $list; + + if(ref($_[0]) eq 'ARRAY') + { + $list = shift; + } + elsif(!ref($_[0])) + { + $list = [ @_ ]; + } + else + { + my $ref = ref($_[0]); + die "String->joinCsv, Param[1] is not ARRAY/ARRRAY-ref. [$ref]\n"; + } + + my $text = join ',', map {(s/"/""/g or /[\r\n,]/) ? qq("$_") : $_} @$list; + + $this->{str} = $text . "\n"; + + $this; +} +sub _utf32be_ucs4 { + my $this = shift; + my $str = shift; + + $str; +} +sub _s2e2 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if (0xa1 <= $c1 && $c1 <= 0xdf) + { + $c2 = $c1; + $c1 = 0x8e; + } + elsif (0x9f <= $c2) + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe0 : 0x60); + $c2 += 2; + } + else + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe1 : 0x61); + $c2 += 0x60 + ($c2 < 0x7f); + } + + $S2E[unpack('n', $c) or unpack('C', $1)] = pack('CC', $c1, $c2); +} +sub z2hKanaK { + my $this = shift; + + if(!defined(%_z2hKanaK)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x80\x81|\xe3\x80\x82|\xe3\x83\xbb|\xe3\x82\x9b|\xe3\x82\x9c|\xe3\x83\xbc|\xe3\x80\x8c|\xe3\x80\x8d|\xe3\x82\xa1|\xe3\x82\xa2|\xe3\x82\xa3|\xe3\x82\xa4|\xe3\x82\xa5|\xe3\x82\xa6|\xe3\x82\xa7|\xe3\x82\xa8|\xe3\x82\xa9|\xe3\x82\xaa|\xe3\x82\xab|\xe3\x82\xad|\xe3\x82\xaf|\xe3\x82\xb1|\xe3\x82\xb3|\xe3\x82\xb5|\xe3\x82\xb7|\xe3\x82\xb9|\xe3\x82\xbb|\xe3\x82\xbd|\xe3\x82\xbf|\xe3\x83\x81|\xe3\x83\x83|\xe3\x83\x84|\xe3\x83\x86|\xe3\x83\x88|\xe3\x83\x8a|\xe3\x83\x8b|\xe3\x83\x8c|\xe3\x83\x8d|\xe3\x83\x8e|\xe3\x83\x8f|\xe3\x83\x92|\xe3\x83\x95|\xe3\x83\x98|\xe3\x83\x9b|\xe3\x83\x9e|\xe3\x83\x9f|\xe3\x83\xa0|\xe3\x83\xa1|\xe3\x83\xa2|\xe3\x83\xa3|\xe3\x83\xa4|\xe3\x83\xa5|\xe3\x83\xa6|\xe3\x83\xa7|\xe3\x83\xa8|\xe3\x83\xa9|\xe3\x83\xaa|\xe3\x83\xab|\xe3\x83\xac|\xe3\x83\xad|\xe3\x83\xaf|\xe3\x83\xb2|\xe3\x83\xb3)/$_z2hKanaK{$1}/eg; + + $this; +} +sub h2zKana +{ + my $this = shift; + + $this->h2zKanaD; + $this->h2zKanaK; + + $this; +} +sub _ucs2_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + for my $uc (unpack("n*", $str)) + { + $result .= $U2T[$uc] ? $U2T[$uc] : + ($U2T[$uc] = ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . + chr(0x80 | ($uc & 0x3F))); + } + + $result; +} +sub z2hAlpha { + my $this = shift; + + if(!defined(%_z2hAlpha)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbc\xa1|\xef\xbc\xa2|\xef\xbc\xa3|\xef\xbc\xa4|\xef\xbc\xa5|\xef\xbc\xa6|\xef\xbc\xa7|\xef\xbc\xa8|\xef\xbc\xa9|\xef\xbc\xaa|\xef\xbc\xab|\xef\xbc\xac|\xef\xbc\xad|\xef\xbc\xae|\xef\xbc\xaf|\xef\xbc\xb0|\xef\xbc\xb1|\xef\xbc\xb2|\xef\xbc\xb3|\xef\xbc\xb4|\xef\xbc\xb5|\xef\xbc\xb6|\xef\xbc\xb7|\xef\xbc\xb8|\xef\xbc\xb9|\xef\xbc\xba|\xef\xbd\x81|\xef\xbd\x82|\xef\xbd\x83|\xef\xbd\x84|\xef\xbd\x85|\xef\xbd\x86|\xef\xbd\x87|\xef\xbd\x88|\xef\xbd\x89|\xef\xbd\x8a|\xef\xbd\x8b|\xef\xbd\x8c|\xef\xbd\x8d|\xef\xbd\x8e|\xef\xbd\x8f|\xef\xbd\x90|\xef\xbd\x91|\xef\xbd\x92|\xef\xbd\x93|\xef\xbd\x94|\xef\xbd\x95|\xef\xbd\x96|\xef\xbd\x97|\xef\xbd\x98|\xef\xbd\x99|\xef\xbd\x9a)/$_z2hAlpha{$1}/eg; + + $this; +} +sub _utf32le_ucs4 { + my $this = shift; + my $str = shift; + + my $result = ''; + foreach my $ch (unpack('V*', $str)) + { + $result .= pack('N', $ch); + } + + $result; +} +sub _utf8_utf16 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $uc; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})/ + $T2U{$1} + or ($T2U{$1} + = ((length($1) == 1) ? pack("n", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("n", (($c1 & 0x1F)<<6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("n", (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F))) : + (length($1) == 4) ? (($c1,$c2,$c3,$c4) = unpack("C4", $1), + ($uc = ((($c1 & 0x07) << 18)|(($c2 & 0x3F) << 12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F)) - 0x10000), + (($uc < 0x100000) ? pack("nn", (($uc >> 10) | 0xd800), (($uc & 0x3ff) | 0xdc00)) : "\0?")) : + "\0?") + ); + /eg; + $str; +} +sub getcode { + my $this = shift; + my $str = shift; + + my $l = length($str); + + if((($l % 4) == 0) + and ($str =~ m/^(?:$RE{BOM4_BE}|$RE{BOM4_LE})/o)) + { + return 'utf32'; + } + if((($l % 2) == 0) + and ($str =~ m/^(?:$RE{BOM2_BE}|$RE{BOM2_LE})/o)) + { + return 'utf16'; + } + + my $str2; + + if(($l % 4) == 0) + { + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{UTF32_BE})//o); + if($str2 eq '') + { + return 'utf32-be'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{UTF32_LE})//o); + if($str2 eq '') + { + return 'utf32-le'; + } + } + + if($str !~ m/[\e\x80-\xff]/) + { + return 'ascii'; + } + + if($str =~ m/$RE{JIS_0208}|$RE{JIS_0212}|$RE{JIS_ASC}|$RE{JIS_KANA}/o) + { + return 'jis'; + } + + if($str =~ m/(?:$RE{E_JSKY})/o) + { + return 'sjis-jsky'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{ASCII}|$RE{EUC_0212}|$RE{EUC_KANA}|$RE{EUC_C})//o); + if($str2 eq '') + { + return 'euc'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{ASCII}|$RE{SJIS_DBCS}|$RE{SJIS_KANA})//o); + if($str2 eq '') + { + return 'sjis'; + } + + my $str3; + $str3 = $str2; + 1 while($str3 =~ s/^(?:$RE{ASCII}|$RE{SJIS_DBCS}|$RE{SJIS_KANA}|$RE{E_IMODE})//o); + if($str3 eq '') + { + return 'sjis-imode'; + } + + $str3 = $str2; + 1 while($str3 =~ s/^(?:$RE{ASCII}|$RE{SJIS_DBCS}|$RE{SJIS_KANA}|$RE{E_DOTI})//o); + if($str3 eq '') + { + return 'sjis-doti'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{UTF8})//o); + if($str2 eq '') + { + return 'utf8'; + } + + return 'unknown'; +} +sub _decodeBase64 +{ + local($^W) = 0; # unpack("u",...) gives bogus warning in 5.00[123] + + my $this = shift; + my $str = shift; + my $res = ""; + + $str =~ tr|A-Za-z0-9+=/||cd; # remove non-base64 chars + if (length($str) % 4) + { + warn("Length of base64 data not a multiple of 4"); + } + $str =~ s/=+$//; # remove padding + $str =~ tr|A-Za-z0-9+/| -_|; # convert to uuencoded format + while ($str =~ /(.{1,60})/gs) + { + my $len = chr(32 + length($1)*3/4); # compute length byte + $res .= unpack("u", $len . $1 ); # uudecode + } + $res; +} +sub sjis_doti +{ + my $this = shift; + $this->_u2sd($this->{str}); +} +sub sjis_jsky +{ + my $this = shift; + $this->_u2sj($this->{str}); +} +sub tag2bin { + my $this = shift; + + $this->{str} =~ s/\&(\#\d+|\#x[a-f0-9A-F]+);/ + (substr($1, 1, 1) eq 'x') ? $this->_ucs4_utf8(pack('N', hex(substr($1, 2)))) : + $this->_ucs4_utf8(pack('N', substr($1, 1))) + /eg; + + $this; +} +sub strcut +{ + my $this = shift; + my $cutlen = shift; + + if(ref($cutlen)) + { + die "String->strcut, Param[1] is Ref.\n"; + } + if($cutlen =~ m/\D/) + { + die "String->strcut, Param[1] must be NUMERIC.\n"; + } + + my $ch_re = '[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}'; + + my $result; + my $line = ''; + my $linelength = 0; + foreach my $c (split(/($ch_re)/, $this->{str})) + { + next if(length($c) == 0); + if($linelength + (length($c) >= 3 ? 2 : 1) > $cutlen) + { + push(@$result, $line); + $line = ''; + $linelength = 0; + } + $linelength += (length($c) >= 3 ? 2 : 1); + $line .= $c; + } + push(@$result, $line); + + $result; +} +sub h2zKanaD { + my $this = shift; + + if(!defined(%_h2zKanaD)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbd\xb3\xef\xbe\x9e|\xef\xbd\xb6\xef\xbe\x9e|\xef\xbd\xb7\xef\xbe\x9e|\xef\xbd\xb8\xef\xbe\x9e|\xef\xbd\xb9\xef\xbe\x9e|\xef\xbd\xba\xef\xbe\x9e|\xef\xbd\xbb\xef\xbe\x9e|\xef\xbd\xbc\xef\xbe\x9e|\xef\xbd\xbd\xef\xbe\x9e|\xef\xbd\xbe\xef\xbe\x9e|\xef\xbd\xbf\xef\xbe\x9e|\xef\xbe\x80\xef\xbe\x9e|\xef\xbe\x81\xef\xbe\x9e|\xef\xbe\x82\xef\xbe\x9e|\xef\xbe\x83\xef\xbe\x9e|\xef\xbe\x84\xef\xbe\x9e|\xef\xbe\x8a\xef\xbe\x9e|\xef\xbe\x8a\xef\xbe\x9f|\xef\xbe\x8b\xef\xbe\x9e|\xef\xbe\x8b\xef\xbe\x9f|\xef\xbe\x8c\xef\xbe\x9e|\xef\xbe\x8c\xef\xbe\x9f|\xef\xbe\x8d\xef\xbe\x9e|\xef\xbe\x8d\xef\xbe\x9f|\xef\xbe\x8e\xef\xbe\x9e|\xef\xbe\x8e\xef\xbe\x9f)/$_h2zKanaD{$1}/eg; + + $this; +} +sub _utf8_ucs2 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}|(.))/ + defined($2)?"\0$2": + $T2U{$1} + or ($T2U{$1} + = ((length($1) == 1) ? pack("n", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("n", (($c1 & 0x1F)<<6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("n", (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F))) : "\0?")) + /eg; + $str; +} +sub sjis_imode +{ + my $this = shift; + $this->_u2si($this->{str}); +} +sub _utf8_ucs4 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}|(.))/ + defined($2) ? "\0\0\0$2" : + (length($1) == 1) ? pack("N", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("N", (($c1 & 0x1F) << 6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("N", (($c1 & 0x0F) << 12)|(($c2 & 0x3F) << 6)| + ($c3 & 0x3F))) : + (length($1) == 4) ? (($c1,$c2,$c3,$c4) = unpack("C4", $1), + pack("N", (($c1 & 0x07) << 18)|(($c2 & 0x3F) << 12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F))) : + (length($1) == 5) ? (($c1,$c2,$c3,$c4,$c5) = unpack("C5", $1), + pack("N", (($c1 & 0x03) << 24)|(($c2 & 0x3F) << 18)| + (($c3 & 0x3f) << 12)|(($c4 & 0x3f) << 6)| + ($c5 & 0x3F))) : + (($c1,$c2,$c3,$c4,$c5,$c6) = unpack("C6", $1), + pack("N", (($c1 & 0x03) << 30)|(($c2 & 0x3F) << 24)| + (($c3 & 0x3f) << 18)|(($c4 & 0x3f) << 12)| + (($c5 & 0x3f) << 6)|($c6 & 0x3F))) + /eg; + + $str; +} +sub _u2sd { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + if(!defined($eu2d)) + { + $eu2d = $this->_getFile('jcode/emoji/eu2d.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : + ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '?' + ) + ), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + ( + $c = substr($eu2d, ($ch - 0x0ff000) * 2, 2), + $c =~ tr,\0,,d, + ($c eq '') ? '?' : $c + ) : + '?' + ) + ) : + '?' + ) + /eg; + $str; + +} +sub get { + my $this = shift; + $this->{str}; +} +sub utf8 +{ + my $this = shift; + $this->{str}; +} +sub hira2kata { + my $this = shift; + + if(!defined(%_hira2kata)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x81\x81|\xe3\x81\x82|\xe3\x81\x83|\xe3\x81\x84|\xe3\x81\x85|\xe3\x81\x86|\xe3\x81\x87|\xe3\x81\x88|\xe3\x81\x89|\xe3\x81\x8a|\xe3\x81\x8b|\xe3\x81\x8c|\xe3\x81\x8d|\xe3\x81\x8e|\xe3\x81\x8f|\xe3\x81\x90|\xe3\x81\x91|\xe3\x81\x92|\xe3\x81\x93|\xe3\x81\x94|\xe3\x81\x95|\xe3\x81\x96|\xe3\x81\x97|\xe3\x81\x98|\xe3\x81\x99|\xe3\x81\x9a|\xe3\x81\x9b|\xe3\x81\x9c|\xe3\x81\x9d|\xe3\x81\x9e|\xe3\x81\x9f|\xe3\x81\xa0|\xe3\x81\xa1|\xe3\x81\xa2|\xe3\x81\xa3|\xe3\x81\xa4|\xe3\x81\xa5|\xe3\x81\xa6|\xe3\x81\xa7|\xe3\x81\xa8|\xe3\x81\xa9|\xe3\x81\xaa|\xe3\x81\xab|\xe3\x81\xac|\xe3\x81\xad|\xe3\x81\xae|\xe3\x81\xaf|\xe3\x81\xb0|\xe3\x81\xb1|\xe3\x81\xb2|\xe3\x81\xb3|\xe3\x81\xb4|\xe3\x81\xb5|\xe3\x81\xb6|\xe3\x81\xb7|\xe3\x81\xb8|\xe3\x81\xb9|\xe3\x81\xba|\xe3\x81\xbb|\xe3\x81\xbc|\xe3\x81\xbd|\xe3\x81\xbe|\xe3\x81\xbf|\xe3\x82\x80|\xe3\x82\x81|\xe3\x82\x82|\xe3\x82\x83|\xe3\x82\x84|\xe3\x82\x85|\xe3\x82\x86|\xe3\x82\x87|\xe3\x82\x88|\xe3\x82\x89|\xe3\x82\x8a|\xe3\x82\x8b|\xe3\x82\x8c|\xe3\x82\x8d|\xe3\x82\x8e|\xe3\x82\x8f|\xe3\x82\x90|\xe3\x82\x91|\xe3\x82\x92|\xe3\x82\x93)/$_hira2kata{$1}/eg; + + $this; +} +sub z2h { + my $this = shift; + + $this->z2hKana; + $this->z2hNum; + $this->z2hAlpha; + $this->z2hSym; + + $this; +} +sub _encodeBase64 +{ + my $this = shift; + my $str = shift; + my $eol = shift; + my $res = ""; + + $eol = "\n" unless defined $eol; + pos($str) = 0; # ensure start at the beginning + while ($str =~ /(.{1,45})/gs) + { + $res .= substr(pack('u', $1), 1); + chop($res); + } + $res =~ tr|` -_|AA-Za-z0-9+/|; # `# help emacs + # fix padding at the end + my $padding = (3 - length($str) % 3) % 3; + $res =~ s/.{$padding}$/'=' x $padding/e if $padding; + # break encoded string into lines of no more than 76 characters each + if (length $eol) + { + $res =~ s/(.{1,76})/$1$eol/g; + } + $res; +} +sub h2zKanaK { + my $this = shift; + + if(!defined(%_h2zKanaK)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbd\xa1|\xef\xbd\xa2|\xef\xbd\xa3|\xef\xbd\xa4|\xef\xbd\xa5|\xef\xbd\xa6|\xef\xbd\xa7|\xef\xbd\xa8|\xef\xbd\xa9|\xef\xbd\xaa|\xef\xbd\xab|\xef\xbd\xac|\xef\xbd\xad|\xef\xbd\xae|\xef\xbd\xaf|\xef\xbd\xb0|\xef\xbd\xb1|\xef\xbd\xb2|\xef\xbd\xb3|\xef\xbd\xb4|\xef\xbd\xb5|\xef\xbd\xb6|\xef\xbd\xb7|\xef\xbd\xb8|\xef\xbd\xb9|\xef\xbd\xba|\xef\xbd\xbb|\xef\xbd\xbc|\xef\xbd\xbd|\xef\xbd\xbe|\xef\xbd\xbf|\xef\xbe\x80|\xef\xbe\x81|\xef\xbe\x82|\xef\xbe\x83|\xef\xbe\x84|\xef\xbe\x85|\xef\xbe\x86|\xef\xbe\x87|\xef\xbe\x88|\xef\xbe\x89|\xef\xbe\x8a|\xef\xbe\x8b|\xef\xbe\x8c|\xef\xbe\x8d|\xef\xbe\x8e|\xef\xbe\x8f|\xef\xbe\x90|\xef\xbe\x91|\xef\xbe\x92|\xef\xbe\x93|\xef\xbe\x94|\xef\xbe\x95|\xef\xbe\x96|\xef\xbe\x97|\xef\xbe\x98|\xef\xbe\x99|\xef\xbe\x9a|\xef\xbe\x9b|\xef\xbe\x9c|\xef\xbe\x9d|\xef\xbe\x9e|\xef\xbe\x9f)/$_h2zKanaK{$1}/eg; + + $this; +} +sub _u2si { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + if(!defined($eu2i)) + { + $eu2i = $this->_getFile('jcode/emoji/eu2i.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : + ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '?' + ) + ), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + ( + $c = substr($eu2i, ($ch - 0x0ff000) * 2, 2), + $c =~ tr,\0,,d, + ($c eq '') ? '?' : $c + ) : + '?' + ) + ) : + '?' + ) + /eg; + $str; + +} +sub _u2sj { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + if(!defined($eu2j)) + { + $eu2j = $this->_getFile('jcode/emoji/eu2j.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : + ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '?' + ) + ), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + ( + $c = substr($eu2j, ($ch - 0x0ff000) * 5, 5), + $c =~ tr,\0,,d, + ($c eq '') ? '?' : $c + ) : + '?' + ) + ) : + '?' + ) + /eg; + + 1 while($str =~ s/($RE{E_JSKY_START})($RE{E_JSKY1})($RE{E_JSKY2}+)$RE{E_JSKY_END}$RE{E_JSKY_START}\2($RE{E_JSKY2})($RE{E_JSKY_END})/$1$2$3$4$5/o); + + $str; + +} +sub h2zAlpha { + my $this = shift; + + if(!defined(%_h2zAlpha)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z|a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)/$_h2zAlpha{$1}/eg; + + $this; +} +sub _s2e { + my $this = shift; + my $str = shift; + + $str =~ s/($RE{SJIS_DBCS}|$RE{SJIS_KANA})/ + $S2E[unpack('n', $1) or unpack('C', $1)] or $this->_s2e2($1) + /geo; + + $str; +} +sub _e2s2 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if ($c1 == 0x8e) + { # SS2 + $E2S[unpack('n', $c)] = chr($c2); + } + elsif ($c1 == 0x8f) + { # SS3 + $E2S[unpack('N', "\0" . $c)] = $CHARCODE{UNDEF_SJIS}; + } + else + { #SS1 or X0208 + if ($c1 % 2) + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x31 : 0x71); + $c2 -= 0x60 + ($c2 < 0xe0); + } + else + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x30 : 0x70); + $c2 -= 2; + } + $E2S[unpack('n', $c)] = pack('CC', $c1, $c2); + } +} +sub _utf16le_utf16 { + my $this = shift; + my $str = shift; + + my $result = ''; + foreach my $ch (unpack('v*', $str)) + { + $result .= pack('n', $ch); + } + + $result; +} +sub _sj2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + if(!defined($ej2u)) + { + $ej2u = $this->_getFile('jcode/emoji/ej2u.dat'); + } + + my $l; + my $j1; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|$RE{E_JSKY}|[\x00-\xff])/ + (length($1) <= 2) ? + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xffff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) : + ( + $l = $1, + $l =~ s,^$RE{E_JSKY_START}($RE{E_JSKY1}),,o, + $j1 = $1, + $uc = '', + $l =~ s!($RE{E_JSKY2})!$uc .= substr($ej2u, (unpack('n', $j1 . $1) - 0x4500) * 4, 4), ''!ego, + $uc =~ tr,\0,,d, + $uc + ) + /eg; + + $str; + +} +sub _s2j { + my $this = shift; + my $str = shift; + + $str =~ s/((?:$RE{SJIS_DBCS}|$RE{SJIS_KANA})+)/ + $this->_s2j2($1) . $ESC{ASC} + /geo; + + $str; +} +sub _s2j2 { + my $this = shift; + my $str = shift; + + $str =~ s/((?:$RE{SJIS_DBCS})+|(?:$RE{SJIS_KANA})+)/ + my $s = $1; + if($s =~ m,^$RE{SJIS_KANA},) + { + $s =~ tr,\xa1-\xdf,\x21-\x5f,; + $ESC{KANA} . $s + } + else + { + $s =~ s!($RE{SJIS_DBCS})! + $S2J[unpack('n', $1)] or $this->_s2j3($1) + !geo; + $ESC{JIS_0208} . $s; + } + /geo; + + $str; +} +sub _s2j3 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if (0x9f <= $c2) + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe0 : 0x60); + $c2 += 2; + } + else + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe1 : 0x61); + $c2 += 0x60 + ($c2 < 0x7f); + } + + $S2J[unpack('n', $c)] = pack('CC', $c1 - 0x80, $c2 - 0x80); +} +sub conv { + my $this = shift; + my $ocode = shift; + my $encode = shift; + my (@option) = @_; + + my $res; + if($ocode eq 'utf8') + { + $res = $this->utf8; + } + elsif($ocode eq 'euc') + { + $res = $this->euc; + } + elsif($ocode eq 'jis') + { + $res = $this->jis; + } + elsif($ocode eq 'sjis') + { + $res = $this->sjis; + } + elsif($ocode eq 'sjis-imode') + { + $res = $this->sjis_imode; + } + elsif($ocode eq 'sjis-doti') + { + $res = $this->sjis_doti; + } + elsif($ocode eq 'sjis-jsky') + { + $res = $this->sjis_jsky; + } + elsif($ocode eq 'ucs2') + { + $res = $this->ucs2; + } + elsif($ocode eq 'ucs4') + { + $res = $this->ucs4; + } + elsif($ocode eq 'utf16') + { + $res = $this->utf16; + } + elsif($ocode eq 'binary') + { + $res = $this->{str}; + } + else + { + die qq(String->conv, Param[1] "$ocode" is error.\n); + } + + if(defined($encode)) + { + if($encode eq 'base64') + { + $res = $this->_encodeBase64($res, @option); + } + else + { + die qq(String->conv, Param[2] "$encode" encode name error.\n); + } + } + + $res; +} +sub _s2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + my $l; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|[\x00-\xff])/ + $S2U{$1} + or ($S2U{$1} = + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xfcff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) + )/eg; + + $str; + +} +sub _j2s { + my $this = shift; + my $str = shift; + + $str =~ s/($RE{JIS_0208}|$RE{JIS_0212}|$RE{JIS_ASC}|$RE{JIS_KANA})([^\e]*)/ + $this->_j2s2($1, $2) + /geo; + + $str; +} +sub h2z { + my $this = shift; + + $this->h2zKana; + $this->h2zNum; + $this->h2zAlpha; + $this->h2zSym; + + $this; +} +sub ucs2 +{ + my $this = shift; + $this->_utf8_ucs2($this->{str}); +} +sub z2hSym { + my $this = shift; + + if(!defined(%_z2hSym)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x80\x80|\xef\xbc\x8c|\xef\xbc\x8e|\xef\xbc\x9a|\xef\xbc\x9b|\xef\xbc\x9f|\xef\xbc\x81|\xef\xbd\x80|\xef\xbc\xbe|\xef\xbc\x8f|\xe3\x80\x9c|\xef\xbd\x9c|\xe2\x80\x9d|\xef\xbc\x88|\xef\xbc\x89|\xef\xbc\xbb|\xef\xbc\xbd|\xef\xbd\x9b|\xef\xbd\x9d|\xef\xbc\x8b|\xe2\x88\x92|\xef\xbc\x9d|\xef\xbc\x9c|\xef\xbc\x9e|\xef\xbf\xa5|\xef\xbc\x84|\xef\xbc\x85|\xef\xbc\x83|\xef\xbc\x86|\xef\xbc\x8a|\xef\xbc\xa0)/$_z2hSym{$1}/eg; + + $this; +} +sub set +{ + my $this = shift; + my $str = shift; + my $icode = shift; + my $encode = shift; + + if(ref($str)) + { + die "String->set, Param[1] is Ref.\n"; + } + if(ref($icode)) + { + die "String->set, Param[2] is Ref.\n"; + } + if(ref($encode)) + { + die "String->set, Param[3] is Ref.\n"; + } + + if(defined($encode)) + { + if($encode eq 'base64') + { + $str = $this->_decodeBase64($str); + } + else + { + die "String->set, Param[3] encode name error.\n"; + } + } + + if(!defined($icode)) + { + $this->{str} = $str; + } + else + { + $icode = lc($icode); + if($icode eq 'auto') + { + $icode = $this->getcode($str); + } + if($icode eq 'utf8') + { + $this->{str} = $str; + } + elsif($icode eq 'ucs2') + { + $this->{str} = $this->_ucs2_utf8($str); + } + elsif($icode eq 'ucs4') + { + $this->{str} = $this->_ucs4_utf8($str); + } + elsif($icode eq 'utf16-be') + { + $this->{str} = $this->_utf16_utf8($this->_utf16be_utf16($str)); + } + elsif($icode eq 'utf16-le') + { + $this->{str} = $this->_utf16_utf8($this->_utf16le_utf16($str)); + } + elsif($icode eq 'utf16') + { + $this->{str} = $this->_utf16_utf8($this->_utf16_utf16($str)); + } + elsif($icode eq 'utf32-be') + { + $this->{str} = $this->_ucs4_utf8($this->_utf32be_ucs4($str)); + } + elsif($icode eq 'utf32-le') + { + $this->{str} = $this->_ucs4_utf8($this->_utf32le_ucs4($str)); + } + elsif($icode eq 'utf32') + { + $this->{str} = $this->_ucs4_utf8($this->_utf32_ucs4($str)); + } + elsif($icode eq 'jis') + { + $this->{str} = $this->_j2s($str); + $this->{str} = $this->_s2u($this->{str}); + } + elsif($icode eq 'euc') + { + $this->{str} = $this->_e2s($str); + $this->{str} = $this->_s2u($this->{str}); + } + elsif($icode eq 'sjis') + { + $this->{str} = $this->_s2u($str); + } + elsif($icode eq 'sjis-imode') + { + $this->{str} = $this->_si2u($str); + } + elsif($icode eq 'sjis-doti') + { + $this->{str} = $this->_sd2u($str); + } + elsif($icode eq 'sjis-jsky') + { + $this->{str} = $this->_sj2u($str); + } + elsif($icode eq 'ascii') + { + $this->{str} = $str; + } + elsif($icode eq 'unknown') + { + $this->{str} = $str; + } + elsif($icode eq 'binary') + { + $this->{str} = $str; + } + else + { + use Carp; + croak "icode error [$icode]"; + } + } + + $this; +} +sub ucs4 +{ + my $this = shift; + $this->_utf8_ucs4($this->{str}); +} +sub z2hNum { + my $this = shift; + + if(!defined(%_z2hNum)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbc\x90|\xef\xbc\x91|\xef\xbc\x92|\xef\xbc\x93|\xef\xbc\x94|\xef\xbc\x95|\xef\xbc\x96|\xef\xbc\x97|\xef\xbc\x98|\xef\xbc\x99)/$_z2hNum{$1}/eg; + + $this; +} +sub _si2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + if(!defined($ei2u)) + { + $ei2u = $this->_getFile('jcode/emoji/ei2u.dat'); + } + + $str =~ s/(\&\#(\d+);)/ + ($2 >= 0xf800 and $2 <= 0xf9ff) ? pack('n', $2) : $1 + /eg; + + my $l; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|$RE{E_IMODE}|[\x00-\xff])/ + $S2U{$1} + or ($S2U{$1} = + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xf800 and $l <= 0xf9ff) ? + ( + $uc = substr($ei2u, ($l - 0xf800) * 4, 4), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xffff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) + )/eg; + + $str; + +} +sub _e2s { + my $this = shift; + my $str = shift; + + $str =~ s/($RE{EUC_KANA}|$RE{EUC_0212}|$RE{EUC_C})/ + $E2S[unpack('n', $1) or unpack('N', "\0" . $1)] or $this->_e2s2($1) + /geo; + + $str; +} +sub jis +{ + my $this = shift; + $this->_s2j($this->sjis); +} +sub _utf32_ucs4 { + my $this = shift; + my $str = shift; + + if($str =~ s/^\x00\x00\xfe\xff//) + { + $str = $this->_utf32be_ucs4($str); + } + elsif($str =~ s/^\xff\xfe\x00\x00//) + { + $str = $this->_utf32le_ucs4($str); + } + else + { + $str = $this->_utf32be_ucs4($str); + } + + $str; +} +sub kata2hira { + my $this = shift; + + if(!defined(%_kata2hira)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x82\xa1|\xe3\x82\xa2|\xe3\x82\xa3|\xe3\x82\xa4|\xe3\x82\xa5|\xe3\x82\xa6|\xe3\x82\xa7|\xe3\x82\xa8|\xe3\x82\xa9|\xe3\x82\xaa|\xe3\x82\xab|\xe3\x82\xac|\xe3\x82\xad|\xe3\x82\xae|\xe3\x82\xaf|\xe3\x82\xb0|\xe3\x82\xb1|\xe3\x82\xb2|\xe3\x82\xb3|\xe3\x82\xb4|\xe3\x82\xb5|\xe3\x82\xb6|\xe3\x82\xb7|\xe3\x82\xb8|\xe3\x82\xb9|\xe3\x82\xba|\xe3\x82\xbb|\xe3\x82\xbc|\xe3\x82\xbd|\xe3\x82\xbe|\xe3\x82\xbf|\xe3\x83\x80|\xe3\x83\x81|\xe3\x83\x82|\xe3\x83\x83|\xe3\x83\x84|\xe3\x83\x85|\xe3\x83\x86|\xe3\x83\x87|\xe3\x83\x88|\xe3\x83\x89|\xe3\x83\x8a|\xe3\x83\x8b|\xe3\x83\x8c|\xe3\x83\x8d|\xe3\x83\x8e|\xe3\x83\x8f|\xe3\x83\x90|\xe3\x83\x91|\xe3\x83\x92|\xe3\x83\x93|\xe3\x83\x94|\xe3\x83\x95|\xe3\x83\x96|\xe3\x83\x97|\xe3\x83\x98|\xe3\x83\x99|\xe3\x83\x9a|\xe3\x83\x9b|\xe3\x83\x9c|\xe3\x83\x9d|\xe3\x83\x9e|\xe3\x83\x9f|\xe3\x83\xa0|\xe3\x83\xa1|\xe3\x83\xa2|\xe3\x83\xa3|\xe3\x83\xa4|\xe3\x83\xa5|\xe3\x83\xa6|\xe3\x83\xa7|\xe3\x83\xa8|\xe3\x83\xa9|\xe3\x83\xaa|\xe3\x83\xab|\xe3\x83\xac|\xe3\x83\xad|\xe3\x83\xae|\xe3\x83\xaf|\xe3\x83\xb0|\xe3\x83\xb1|\xe3\x83\xb2|\xe3\x83\xb3)/$_kata2hira{$1}/eg; + + $this; +} +sub _ucs4_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + for my $uc (unpack("N*", $str)) + { + $result .= ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x10000) ? chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x200000) ? chr(0xF0 | ($uc >> 18)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x4000000) ? chr(0xF8 | ($uc >> 24)) . chr(0x80 | (($uc >> 18) & 0x3F)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xFC | ($uc >> 30)) . chr(0x80 | (($uc >> 24) & 0x3F)) . chr(0x80 | (($uc >> 18) & 0x3F)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)); + } + + $result; +} +sub split_csv { + my $this = shift; + + $this->splitCsv(@_); +} +sub _utf16_utf16 { + my $this = shift; + my $str = shift; + + if($str =~ s/^\xfe\xff//) + { + $str = $this->_utf16be_utf16($str); + } + elsif($str =~ s/^\xff\xfe//) + { + $str = $this->_utf16le_utf16($str); + } + else + { + $str = $this->_utf16be_utf16($str); + } + + $str; +} +sub _sd2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + if(!defined($ed2u)) + { + $ed2u = $this->_getFile('jcode/emoji/ed2u.dat'); + } + + $str =~ s/(\&\#(\d+);)/ + ($2 >= 0xf000 and $2 <= 0xf4ff) ? pack('n', $2) : $1 + /eg; + + my $l; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|$RE{E_DOTI}|[\x00-\xff])/ + $S2U{$1} + or ($S2U{$1} = + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xf000 and $l <= 0xf4ff) ? + ( + $uc = substr($ed2u, ($l - 0xf000) * 4, 4), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xffff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) + )/eg; + + $str; + +} +sub sjis +{ + my $this = shift; + $this->_u2s($this->{str}); +} +sub _utf16be_utf16 { + my $this = shift; + my $str = shift; + + $str; +} +sub h2zSym { + my $this = shift; + + if(!defined(%_h2zSym)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\x20|\x21|\x22|\x23|\x24|\x25|\x26|\x27|\x28|\x29|\x2a|\x2b|\x2c|\x2d|\x2e|\x2f|\x3a|\x3b|\x3c|\x3d|\x3e|\x3f|\x40|\x5b|\x5c|\x5d|\x5e|\x60|\x7b|\x7c|\x7d|\x7e)/$_h2zSym{$1}/eg; + + $this; +} + + diff --git a/cpan/dist/Unicode-Japanese/Japanese.xs b/cpan/dist/Unicode-Japanese/Japanese.xs new file mode 100644 index 00000000..625c1d0a --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Japanese.xs @@ -0,0 +1,172 @@ + +/* $Id: Japanese.xs,v 1.9 2002/06/30 23:15:45 hio Exp $ */ + +#include "Japanese.h" + +EXTERN_C SV* test(SV* str); + + +MODULE = Unicode::Japanese PACKAGE = Unicode::Japanese + +#========================# +# SJIS <=> utf8 # +#========================# + +SV* +_s2u(this_,str) + SV* str; +CODE: + RETVAL = xs_sjis_utf8(str); +OUTPUT: + RETVAL + +SV* +_u2s(this_,str) + SV* str; +CODE: + RETVAL = xs_utf8_sjis(str); +OUTPUT: + RETVAL + +#========================# +# getcode # +#========================# + +SV* +getcode(this_,str) + SV* str; +CODE: + RETVAL = xs_getcode(str); +OUTPUT: + RETVAL + +#========================# +# SJIS <=> EUCJP # +#========================# + +SV* +_s2e(this_,str) + SV* str; +CODE: + RETVAL = xs_sjis_eucjp(str); +OUTPUT: + RETVAL + +SV* +_e2s(this_,str) + SV* str; +CODE: + RETVAL = xs_eucjp_sjis(str); +OUTPUT: + RETVAL + +#========================# +# SJIS <=> JIS # +#========================# + +SV* +_s2j(this_,str) + SV* str; +CODE: + RETVAL = xs_sjis_jis(str); +OUTPUT: + RETVAL + +SV* +_j2s(this_,str) + SV* str; +CODE: + RETVAL = xs_jis_sjis(str); +OUTPUT: + RETVAL + + +#========================# +# SJIS(i-mode) <=> UTF8 # +#========================# + +SV* +_si2u(this_,str) + SV* str; +CODE: + RETVAL = xs_sjis_imode_utf8(str); +OUTPUT: + RETVAL + +SV* +_u2si(this_,str) + SV* str; +CODE: + RETVAL = xs_utf8_sjis_imode(str); +OUTPUT: + RETVAL + +#========================# +# SJIS(j-sky) <=> UTF8 # +#========================# + +SV* +_sj2u(this_,str) + SV* str; +CODE: + RETVAL = xs_sjis_jsky_utf8(str); +OUTPUT: + RETVAL + +SV* +_u2sj(this_,str) + SV* str; +CODE: + RETVAL = xs_utf8_sjis_jsky(str); +OUTPUT: + RETVAL + +#========================# +# SJIS(dot-i) <=> UTF8 # +#========================# + +SV* +_sd2u(this_,str) + SV* str; +CODE: + RETVAL = xs_sjis_doti_utf8(str); +OUTPUT: + RETVAL + +SV* +_u2sd(this_,str) + SV* str; +CODE: + RETVAL = xs_utf8_sjis_doti(str); +OUTPUT: + RETVAL + +#========================# +# ucs2 <=> utf8 # +#========================# + +SV* +_ucs2_utf8(this_,str) + SV* str; +CODE: + RETVAL = xs_ucs2_utf8(str); +OUTPUT: + RETVAL + +SV* +_utf8_ucs2(this_,str) + SV* str; +CODE: + RETVAL = xs_utf8_ucs2(str); +OUTPUT: + RETVAL + +#=======================# +# memory mapped file # +#=======================# + +void +do_memmap() + +void +do_memunmap() diff --git a/cpan/dist/Unicode-Japanese/Japanese/emoji.dat b/cpan/dist/Unicode-Japanese/Japanese/emoji.dat Binary files differnew file mode 100644 index 00000000..11b9be2e --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Japanese/emoji.dat diff --git a/cpan/dist/Unicode-Japanese/Japanese/u2s-s2u.dat b/cpan/dist/Unicode-Japanese/Japanese/u2s-s2u.dat Binary files differnew file mode 100644 index 00000000..b5a8ae85 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Japanese/u2s-s2u.dat diff --git a/cpan/dist/Unicode-Japanese/MANIFEST b/cpan/dist/Unicode-Japanese/MANIFEST new file mode 100644 index 00000000..8c74bfa2 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/MANIFEST @@ -0,0 +1,39 @@ +Changes +Japanese/u2s-s2u.dat sjis<=>utf8変換テーブル +Japanese/emoji.dat i-mode/j-sky/dot-i<=>utf8変換テーブル +Japanese.pm Unicode::Japanese パッケージ +Japanese.xs Unicode::Japanese XS宣言 +Japanese.h XS-Unicode::Japanese の各種前方宣言 +Makefile.PL +MANIFEST +MANIFEST.noxs +conv.cpp sjis<=>utf8 変換コード +eucjp.cpp sjis<=>eucjp 変換コード +getcode.cpp getcode +getcode.h +getcode_map.cpp getcode用状態遷移テーブル +jis.cpp sjis<=>jis 変換コード +memmap.cpp データファイルのメモリマップコード +sjis.cpp sjis有効判定テーブル +sjis.h sjis有効判定テーブル 前方宣言 +sjis_doti.cpp sjis(dot-i)<=>utf8 変換コード +sjis_imode.cpp sjis(i-mode)<=>utf8 変換コード +sjis_jsky.cpp sjis(j-sky)<=>utf8 変換コード +str.h SV* 操作用クラス +ucs2_utf8.cpp +t/allchar_utf8.pl +t/base.t +t/emoji.t +t/fromutf8.t +t/hirakata.t +t/illlet.t +t/toutf8.t +t/util.t +t/z2h.t +t/getcode.t +t/outrange.t +t/random.pl +t/random_redo.pl +t/ucs2_utf8.pl +t/verify_sjis_ucs2.pl +xs_test.c xsコンパイルテスト用コード diff --git a/cpan/dist/Unicode-Japanese/MANIFEST.noxs b/cpan/dist/Unicode-Japanese/MANIFEST.noxs new file mode 100644 index 00000000..bde72382 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/MANIFEST.noxs @@ -0,0 +1,20 @@ +Changes +Japanese.pm Unicode::Japanese パッケージ +Makefile.PL +MANIFEST +MANIFEST.noxs +t/allchar_utf8.pl +t/base.t +t/emoji.t +t/fromutf8.t +t/hirakata.t +t/illlet.t +t/toutf8.t +t/util.t +t/z2h.t +t/getcode.t +t/outrange.t +t/random.pl +t/random_redo.pl +t/ucs2_utf8.pl +t/verify_sjis_ucs2.pl diff --git a/cpan/dist/Unicode-Japanese/Makefile.PL b/cpan/dist/Unicode-Japanese/Makefile.PL new file mode 100644 index 00000000..5b921284 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/Makefile.PL @@ -0,0 +1,189 @@ + +use vars qw(%param %param_xs $ostype); + +init_param(); +add_ccflags('-Werror'); +#add_ccflags('-DNDEBUG'); # uncomment for release, +#include_test(); +do_mediate(); + +use ExtUtils::MakeMaker; +WriteMakefile(%param,%param_xs); + +sub include_test +{ + $param_xs{'CCFLAGS'} .= '-g -DTEST=1'; + $param_xs{'OBJECT'} .= ' test.o'; +} + +# +# mediate.h/mediate.cpp の作成 +sub do_mediate +{ + # check @ARGV + my @arg; + @ARGV = grep{ !(@arg-push(@arg,/^(PurePerl|NoXS)(=(0|1|yes|no|y|n))?$/i)) } @ARGV; + if( @arg ) + { + my $v = $arg[1] ? $arg[2] : '1'; + if( $v eq '1' || $v eq 'y' || $v eq 'yes' ) + { + print "install with PurePerl mode\n"; + disable_xs(); + return; + } + } + $ostype = `uname -s` || ''; + if( $ostype =~ /(Linux)/ ) + { + $ostype = $1; + }elsif( $ostype =~ /(BSD)/ ) + { + $ostype = $1; + $hdr = "#include <fcntl.h>\n\n"; + }elsif( $ostype =~ /(SunOS)/ ) + { + $ostype = $1; + $hdr = "#include <fcntl.h>\n\n"; + }elsif( !$ostype ) + { + print "Here environment cannot use XS.\n"; + disable_xs(); + return; + } + + if( $] < 5.006 ) + { + /`/; + $hdr .= <<'EOF'; +#define aTHX_ +#define pTHX_ +#define dTHX_ + +#define get_av(var_name,create_flag) perl_get_av(var_name,create_flag); + +#ifndef newSVpvn +#define newSVpvn(str,len) newSVpv(str,len) +#endif + +EOF + if( $] < 5.005 ) + { + # PL_naも作ってもいいけどむしろ使わない方向で. + $hdr .= <<'EOF'; +#ifndef PL_sv_undef +#define PL_sv_undef sv_undef +#endif +EOF + } # if( $] < 5.005 ) + /`/; + } # if( $] < 5.006 ) + + use FileHandle; + my $fh_hdr = new FileHandle('>mediate.h'); + my $fh_cpp = new FileHandle('>mediate.cpp'); + if( !$fh_hdr || !$fh_cpp ) + { + die 'cannot open mediate.h or .cpp for output'; + } + + $hdr = "/* nothing is needed. */\n" unless( $hdr ); + $cpp = "/* nothing is needed. */\n" unless( $cpp ); + + print $fh_cpp <<EOF; + +#include "Japanese.h" + +$cpp +EOF + + print $fh_hdr <<EOF; +#ifndef MEDIATE_H__ +#define MEDIATE_H__ + +$hdr +#endif +EOF + + # compile check. + my $cmd = "$param_xs{CC} -shared -c xs_test.c"; + print "Checking XS Compiling... : $cmd\n"; + my $rc = 0xffff & system($cmd); + if( $rc ) + { + if( $rc == 0xffff ) + { + print "Here environment cannot use XS.\n"; + print "Because : command failed : $!\n"; + disable_xs(); + return; + }elsif( $rc > 0x80 ) + { + print "Here environment cannot use XS.\n"; + print "Because : ran with non-zero exit status $rc\n"; + disable_xs(); + return; + }elsif( $rc & 0x80 ) + { + $rc &= ~0x80; + print "Compile failed..\n"; + print "Because : ran with coredump from signal $rc\n"; + exit 1; + }else + { + print "Compile failed..\n"; + print "Because : ran with signal $rc\n"; + exit 1; + } + } + unlink('xs_test.o'); +} + +sub disable_xs +{ + use ExtUtils::Manifest; + $ExtUtils::Manifest::MANIFEST = 'MANIFEST.noxs'; + %param_xs = ('linkext'=>{'LINKTYPE'=>''}); + #rename('Japanese','Japanese-notused'); +} + +sub init_param +{ + my $cc = 'g++'; + my $ld = $cc; + + @ARGV = grep{ /^CC=(.*)/ ? !($cc=$1) : 1 } @ARGV; + + my $object = join(' ',qw( Japanese.o memmap.o conv.o ucs2_utf8.o + getcode.o getcode_map.o sjis.o eucjp.o jis.o + sjis_imode.o sjis_jsky.o sjis_doti.o + mediate.o + ) ); + + %param = ( + 'NAME' => 'Unicode::Japanese', + 'VERSION_FROM' => 'Japanese.pm', # finds $VERSION + 'PREREQ_PM' => {}, # e.g., Module::Name => 1.1 + 'LIBS' => [''], # e.g., '-lm' + 'DEFINE' => '', # e.g., '-DHAVE_SOMETHING' + 'INC' => '', # e.g., '-I/usr/include/other' + ); + %param_xs = ( + 'EXE_FILES' => ['Japanese/u2s-s2u.dat','Japanese/emoji.dat'], + 'INSTALLSCRIPT' => '$(INST_LIBDIR)', + 'CC' => $cc, + 'CCFLAGS' => '-Wall', + 'LD' => $ld, + 'OBJECT' => $object, + 'XS' => {'Japanese.xs','Japanese.c',}, + 'XSOPT'=>'-C++', + 'XSPROTOARG'=>'-prototypes', + + 'clean' => { FILES => 'mediate.{cpp,h}' }, + ); +} + +sub add_ccflags +{ + $param_xs{'CCFLAGS'} = join(' ',$param_xs{'CC_FLAGS'},@_); +} diff --git a/cpan/dist/Unicode-Japanese/conv.cpp b/cpan/dist/Unicode-Japanese/conv.cpp new file mode 100644 index 00000000..857d0fca --- /dev/null +++ b/cpan/dist/Unicode-Japanese/conv.cpp @@ -0,0 +1,265 @@ + +// $Id: conv.cpp,v 1.15 2002/07/04 04:52:29 hio Exp $ + +#include <stdio.h> +#include "Japanese.h" +#include <netinet/in.h> + +#define DISP_S2U 0 +#define DISP_U2S 0 + +#if DISP_U2S +#define ECHO_U2S(arg) fprintf arg +#define ON_U2S(cmd) cmd +#else +#define ECHO_U2S(arg) +#define ON_U2S(cmd) +#endif + +EXTERN_C +SV* +xs_sjis_utf8(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + STRLEN src_len; + unsigned char* src = (unsigned char*)SvPV(sv_str,src_len); + int len = sv_len(sv_str); + +#if DISP_S2U + fprintf(stderr,"Unicode::Japanese::(xs)sjis_utf8\n",len); + bin_dump("in ",src,len); +#endif + + //asm volatile(".int 3"); + SV_Buf result(len*3/2+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + const unsigned char* ptr; + if( src[0]<0x80 ) + { // ASCII + //fprintf(stderr,"ascii: %02x\n",src[0]); + result.append(*src++); + continue; + }else if( 0xa1<=src[0] && src[0]<=0xdf ) + { // 半角カナ + //fprintf(stderr,"kana": %02x\n",src[0]); + ptr = (unsigned char*)&g_s2u_table[src[0]]; + ++src; + }else if( ((0x81<=src[0] && src[0]<=0x9f) || (0xe0<=src[0] && src[0]<=0xfc) ) + && (0x40<=src[1] && src[1]<=0xfc && src[1]!=0x7f) ) + { // 2バイト文字 + register const unsigned short sjis = ntohs(*(unsigned short*)src); + //fprintf(stderr,"sjis: %04x\n",sjis); + ptr = (unsigned char*)&g_s2u_table[sjis]; + src += 2; + }else + { // 不明 + //fprintf(stderr,"unknown: %02x\n",src[0]); + result.append('?'); + ++src; + continue; + } + + //fprintf(stderr,"utf8-char : %02x %02x %02x %02x\n",ptr[0],ptr[1],ptr[2],ptr[3]); + if( ptr[3] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",4); + result.append_ch4(*(int*)ptr); + }else if( ptr[2] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",3); + result.append_ch3(*(int*)ptr); + }else if( ptr[1] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",2); + result.append_ch2(*(short*)ptr); + }else if( ptr[0] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",1); + result.append(*ptr); + }else + { + result.append('?'); + } + } +#if DISP_S2U + bin_dump("out",result.getBegin(),result.getLength()); +#endif + result.setLength(); + + return result.getSv(); +} + +EXTERN_C +SV* +xs_utf8_sjis(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + ECHO_U2S((stderr,"Unicode::Japanese::(xs)utf8_sjis\n")); + ON_U2S( bin_dump("in ",src,len) ); + + SV_Buf result(len+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + if( *src<=0x7f ) + { + // ASCIIはまとめて追加〜 + int len = 1; + while( src+len<src_end && src[len]<=0x7f ) + { + ++len; + } + result.append(src,len); + src+=len; + continue; + } + // utf8をucsに変換 + // utf8の1文字の長さチェック + int utf8_len; + if( 0xc0<=*src && *src<=0xdf ) + { + utf8_len = 2; + }else if( 0xe0<=*src && *src<=0xef ) + { + utf8_len = 3; + }else if( 0xf0<=*src && *src<=0xf7 ) + { + utf8_len = 4; + }else if( 0xf8<=*src && *src<=0xfb ) + { + utf8_len = 5; + }else if( 0xfc<=*src && *src<=0xfd ) + { + utf8_len = 6; + }else + { + result.append('?'); + ++src; + continue; + } + // 長さ足りてるかチェック + if( src+utf8_len-1>=src_end ) + { + ECHO_U2S((stderr," no enough buffer, here is %d, need %d\n",src_end-src,utf8_len)); + result.append('?'); + ++src; + continue; + } + // 2バイト目以降が正しい文字範囲か確認 + bool succ = true; + for( int i=1; i<utf8_len; ++i ) + { + if( src[i]<0x80 || 0xbf<src[i] ) + { + ECHO_U2S((stderr," at %d, char out of range\n",i)); + succ = false; + break; + } + } + if( !succ ) + { + result.append('?'); + ++src; + continue; + } + // utf8からucsのコードを算出 + ECHO_U2S((stderr,"utf8-charlen: [%d]\n",utf8_len)); + unsigned int ucs; + switch(utf8_len) + { + case 2: + { + ucs = ((src[0] & 0x1F)<<6)|(src[1] & 0x3F); + break; + } + case 3: + { + ucs = ((src[0] & 0x0F)<<12)|((src[1] & 0x3F)<<6)|(src[2] & 0x3F); + break; + } + case 4: + { + ucs = ((src[0] & 0x07)<<18)|((src[1] & 0x3F)<<12)| + ((src[2] & 0x3f) << 6)|(src[3] & 0x3F); + break; + } + case 5: + { + ucs = ((src[0] & 0x03) << 24)|((src[1] & 0x3F) << 18)| + ((src[2] & 0x3f) << 12)|((src[3] & 0x3f) << 6)| + (src[4] & 0x3F); + break; + } + case 6: + { + ucs = ((src[0] & 0x03) << 30)|((src[1] & 0x3F) << 24)| + ((src[2] & 0x3f) << 18)|((src[3] & 0x3f) << 12)| + ((src[4] & 0x3f) << 6)|(src[5] & 0x3F); + break; + } + default: + { + // NOT REACH HERE + ECHO_U2S((stderr,"invalid utf8-length: %d\n",utf8_len)); + ucs = '?'; + } + } + + if( 0x0f0000<=ucs && ucs<=0x0fffff ) + { // 絵文字判定(sjis) + result.append('?'); + assert(utf8_len>=4); + src += utf8_len; + continue; + } + + if( ucs & ~0xFFFF ) + { // ucs2の範囲外 (ucs4の範囲) + result.append_entityref(ucs); + src += utf8_len; + continue; + } + + // ucs => sjis + ECHO_U2S((stderr,"ucs2 [%04x]\n",ucs)); + const unsigned short sjis = g_u2s_table[ucs]; + ECHO_U2S((stderr,"sjis [%04x]\n",ntohs(sjis) )); + + if( sjis || !ucs ) + { // 対応文字がある時とucs=='\0'の時 + if( sjis & 0xff00 ) + { + result.append_ch2(sjis); + }else + { + result.append((unsigned char)sjis); + } + }else if( ucs<=0x7F ) + { + result.append((unsigned char)ucs); + }else + { + result.append_entityref(ucs); + } + src += utf8_len; + //bin_dump("now",dst_begin,dst-dst_begin); + } /* for */ + + ON_U2S( bin_dump("out",result.getBegin(),result.getLength()) ); + result.setLength(); + + return result.getSv(); +} diff --git a/cpan/dist/Unicode-Japanese/eucjp.cpp b/cpan/dist/Unicode-Japanese/eucjp.cpp new file mode 100644 index 00000000..030e77c8 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/eucjp.cpp @@ -0,0 +1,206 @@ + +#include "Japanese.h" +#include "sjis.h" + +#ifdef TEST +#define DISP_E2S 0 +#define DISP_S2E 0 +#endif + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// sjis=>eucjp変換 +EXTERN_C +SV* +xs_sjis_eucjp(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)sjis_eucjp\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + switch(chk_sjis[*src]) + { + case CHK_SJIS_THROUGH: + { + const unsigned char* start = src; + while( ++src<src_end && chk_sjis[*src]==CHK_SJIS_THROUGH ); + result.append(start,src-start); + continue; + } + case CHK_SJIS_C: + { + if( src+2-1<src_end && 0x40<=src[1] && src[1]<=0xfc && src[1]!=0x7f ) + { + unsigned char tmp[2]; + if( 0x9f <= src[1] ) + { + tmp[0] = src[0]*2 - (src[0]>=0xe0 ? 0xe0 : 0x60); + tmp[1] = src[1] + 2; + }else + { + tmp[0] = src[0]*2 - (src[0]>=0xe0 ? 0xe1 : 0x61); + tmp[1] = src[1] + 0x60 + (src[1] < 0x7f); + } + result.append(tmp,2); + src += 2; + continue; + } + break; + } + case CHK_SJIS_KANA: + { + unsigned char tmp[2] = { 0x8e, src[0], }; + result.append(tmp,2); + ++src; + continue; + } + default: + { +#ifdef TEST + fprintf(stderr,"xs_sjis_eucjp, unknown check-code[%02x] on char-code[%05x]\n",chk_sjis[*src],*src); +#endif + result.append(*src++); + } + } //switch + + // invalid char + result.append(*src++); + + } //while + + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// eucjp=>sjis変換文字判定 +// 1:EUCJP:0212, 3:EUCJP:C 4:EUCJP:KANA +#define CHK_EUCJP_THROUGH 0 +#define CHK_EUCJP_0212 1 +#define CHK_EUCJP_C 3 +#define CHK_EUCJP_KANA 4 +static const unsigned char chk_eucjp[256] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 3 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 5 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 6 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 7 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, // 8 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9 + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // a + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // b + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // c + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // d + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // e + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, // f +}; + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// eucjp=>sjis変換 +EXTERN_C +SV* +xs_eucjp_sjis(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + +#if DISP_E2S + fprintf(stderr,"Unicode::Japanese::(xs)eucjp_sjis\n",len); + bin_dump("in ",src,len); +#endif + + SV_Buf result(len); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + switch(chk_eucjp[*src]) + { + case CHK_EUCJP_THROUGH: + { + const unsigned char* start = src; + while( ++src<src_end && chk_eucjp[*src]==CHK_EUCJP_THROUGH ); + result.append(start,src-start); + continue; + } + case CHK_EUCJP_0212: + { + if( src+3-1<src_end ) + { + result.append(UNDEF_SJIS,UNDEF_SJIS_LEN); + src += 3; + continue; + } + break; + } + case CHK_EUCJP_C: + { + if( src+2-1<src_end && 0xa1<=src[1] && src[1]<=0xfe ) + { + unsigned char tmp[2]; + if( src[0]%2 ) + { + tmp[0] = (src[0]>>1) + (src[0] < 0xdf ? 0x31 : 0x71); + tmp[1] = src[1] - ( 0x60 + (src[1] < 0xe0) ); + }else + { + tmp[0] = (src[0]>>1) + (src[0] < 0xdf ? 0x30 : 0x70); + tmp[1] = src[1] - 2; + } + result.append(tmp,2); + src += 2; + continue; + } + break; + } + case CHK_EUCJP_KANA: + { + if( src+2-1<src_end && 0xa1<=src[1] && src[1]<=0xdf ) + { + result.append(src[1]); + src += 2; + continue; + } + break; + } + default: + { +#ifdef TEST + fprintf(stderr,"xs_eucjp_sjis, unknown check-code[%02x] on char-code[%05x]\n",chk_sjis[*src],*src); +#endif + } + } //switch + + // invalid char + result.append(*src++); + + } //while + +#if DISP_E2S + bin_dump("out",result.getBegin(),result.getLength()); +#endif + + result.setLength(); + + return result.getSv(); +} diff --git a/cpan/dist/Unicode-Japanese/getcode.cpp b/cpan/dist/Unicode-Japanese/getcode.cpp new file mode 100644 index 00000000..129c3668 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/getcode.cpp @@ -0,0 +1,317 @@ + +// $Id: getcode.cpp,v 1.4 2002/07/01 00:17:02 hio Exp $ + +#include "Japanese.h" +#include "getcode.h" + +#define GC_DISP 0 + +// 文字コード定数 +enum charcode_t +{ + cc_unknown, + cc_ascii, + cc_sjis, + cc_eucjp, + cc_jis, + cc_utf8, + cc_utf32, + cc_utf32_be, + cc_utf32_le, + cc_sjis_jsky, + cc_sjis_imode, + cc_sjis_doti, + cc_last, +}; + +// 文字コード名文字列(SV*) +#define new_CC_UNKNOWN() newSVpvn("unknown",7) +#define new_CC_ASCII() newSVpvn("ascii", 5) +#define new_CC_SJIS() newSVpvn("sjis", 4) +#define new_CC_JIS() newSVpvn("jis", 3) +#define new_CC_EUCJP() newSVpvn("euc", 3) +#define new_CC_UTF8() newSVpvn("utf8", 4) +#define new_CC_UTF16() newSVpvn("utf16", 5) +#define new_CC_UTF32() newSVpvn("utf32", 5) +#define new_CC_UTF32_BE() newSVpvn("utf32-be",8) +#define new_CC_UTF32_LE() newSVpvn("utf32-le",8) +#define new_CC_SJIS_JSKY() newSVpvn("sjis-jsky",9) +#define new_CC_SJIS_IMODE() newSVpvn("sjis-imode",10) +#define new_CC_SJIS_DOTI() newSVpvn("sjis-doti",9) + +// +#define RE_BOM2_BE "\xfe\xff" +#define RE_BOM2_LE "\xff\xfe" +#define RE_BOM4_BE "\x00\x00\xfe\xff" +#define RE_BOM4_LE "\xff\xfe\x00\x00" + +#if defined(TEST) || 1 +// 文字コード定数を文字コード名に +static const char* charcodeToStr(charcode_t code) +{ + switch(code) + { + case cc_unknown: return "unknown"; + case cc_ascii: return "ascii"; + case cc_sjis: return "sjis"; + case cc_eucjp: return "eucjp"; + case cc_jis: return "jis"; + case cc_utf8: return "utf8"; + case cc_utf32: return "utf32"; + case cc_utf32_be: return "utf32-be"; + case cc_utf32_le: return "utf32-le"; + case cc_sjis_jsky: return "sjis-jsky"; + case cc_sjis_imode: return "sjis-imode"; + case cc_sjis_doti: return "sjis-doti"; + } + return NULL; +} +#endif +#ifdef TEST +DECL_MAP_MODE(ascii,1) = { "ascii", }; +DECL_MAP_MODE(eucjp,5) = +{ "eucjp", "0212:3.1","0212:3.2","c:2.1","kana:2.1",}; +DECL_MAP_MODE(sjis,2) = { "sjis","c:2.1", }; +DECL_MAP_MODE(jis,10) = +{ + "jis","jis#1","jis#2","jis#3","jis#4","jis#5","jis#6", + "jis#7","jis#loop1","jis#loop2", +}; +DECL_MAP_MODE(utf8,6) = +{ + "utf8", + "u8:6.1","u8:6.2","u8:6.3","u8:6.4","u8:6.5", +}; +DECL_MAP_MODE(utf32_be,4) = +{ + "utf32-be","utf32-be:4:1","utf32-be:4:2","utf32-be:4:3", +}; +DECL_MAP_MODE(utf32_le,4) = +{ + "utf32-le","utf32-le:4:1","utf32-le:4:2","utf32-le:4:3", +}; +DECL_MAP_MODE(sjis_jsky,5) = +{ + "sjis","c:2.1", + "jsky:start:1","jsky:start:2","jsky:code1", +}; +DECL_MAP_MODE(sjis_imode,4) = +{ + "sjis","c:2.1", + "imode1:1","imode2:1", +}; +DECL_MAP_MODE(sjis_doti,7) = +{ + "sjis","c:2.1", + "doti1:1", "doti2:1", "doti3:1", "doti4:1", "doti5:1", +}; +#endif + +// 文字コード判定時に使用する構造体 +struct CodeCheck +{ + charcode_t code; + const unsigned char* base; + const unsigned char* table; +#ifdef TEST + const char** msg; +#endif +}; + +// 文字コード判定の初期状態 +#ifndef TEST +#define GEN_CODE(name) \ + { cc_##name, (const unsigned char*)map_##name, (const unsigned char*)map_##name, } +#else +#define GEN_CODE(name) \ + { cc_##name, (const unsigned char*)map_##name, (const unsigned char*)map_##name, mode_##name, } +#endif +#define cc_tmpl_max 10 +const CodeCheck cc_tmpl[] = +{ + GEN_CODE(ascii), + GEN_CODE(eucjp), + GEN_CODE(sjis), + GEN_CODE(jis), + GEN_CODE(utf8), + GEN_CODE(utf32_be), + GEN_CODE(utf32_le), + GEN_CODE(sjis_jsky), + GEN_CODE(sjis_imode), + GEN_CODE(sjis_doti), +}; + +// 判定結果の構造体 +struct CodeResult +{ + charcode_t code; + int begin; + int len; +}; + +// 複数候補から1つを選択 +int choice_one(CodeCheck* check, int cc_max) +{ + charcode_t order[cc_tmpl_max] = + { + cc_utf32_be, + cc_utf32_le, + cc_ascii, + cc_jis, + cc_eucjp, + cc_sjis, + cc_sjis_jsky, + cc_sjis_imode, + cc_sjis_doti, + cc_utf8, + }; + for( int cc=0; cc<cc_tmpl_max; ++cc ) + { + for( int i=0; i<cc_max; ++i ) + { + if( check[i].code==order[cc] ) + { + return i; + } + } + } + return 0; +} + +// getcode関数 +SV* xs_getcode(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return new_SV_UNDEF(); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + const unsigned char* src_end = src+len; + if( len==0 ) + { + return new_CC_UNKNOWN(); + } + if( (len%4)==0 && len>=4 && + ( memcmp(src,RE_BOM4_BE,4)==0 || memcmp(src,RE_BOM4_LE,4)==0 ) ) + { + return new_CC_UTF32(); + } + if( (len%2)==0 && len>=2 && + ( memcmp(src,RE_BOM2_BE,2)==0 || memcmp(src,RE_BOM2_LE,2)==0 ) ) + { + return new_CC_UTF16(); + } + + //fprintf(stderr,"Unicode::Japanese::(xs)getcode[%d]\n",len); + //fprintf(stderr,">>%s<<\n",src); + //bin_dump("in ",src,len); + + //asm volatile(".int 3"); + //SV_Buf result(len*3/2+4); + + CodeCheck check[cc_tmpl_max]; + memcpy(check,cc_tmpl,sizeof(cc_tmpl)); + int cc_max = cc_tmpl_max; + + for( ; src<src_end; ++src ) + { +#if TEST && GC_DISP + fprintf(stderr,"[%d] %d (0x%02x)\n",len-(src_end-src),*src,*src); +#endif + // 遷移を1つ進める〜 + int invalids = 0; + for( int i=0; i<cc_max; ++i ) + { + int nxt = check[i].table[*src]; +#if TEST && GC_DISP + fprintf(stderr," %s : %d (%s)\n",charcodeToStr(check[i].code),nxt,nxt!=map_invalid?check[i].msg[nxt]:"invalid"); +#endif + if( nxt!=map_invalid ) + { + check[i].table = check[i].base+nxt*256; + }else + { + ++invalids; + check[i].table = NULL; + } + } + if( invalids==0 ) + { // 全部継続 + continue; + }else if( cc_max-invalids>0 ) + { // まだあり〜 + int rd = 0; + int wr = 0; + for( ;rd<cc_max; ++rd ) + { + if( check[rd].table ) + { + if( rd!=wr ) + { + check[wr] = check[rd]; + } + ++wr; + } + } + cc_max = wr; + }else + { // 全部だめ〜 + return new_CC_UNKNOWN(); + break; + } + } + + int wr = 0; + for( int i=0; i<cc_max; ++i ) + { + if( check[i].table == check[i].base ) + { + if( wr!=i ) + { + check[wr] = check[i]; + } + ++wr; + } + } + cc_max = wr; + +#if TEST && GC_DISP + fprintf(stderr,"<availables>\n"); + for( int i=0; i<cc_max; ++i ) + { + fprintf(stderr," %s\n",charcodeToStr(check[i].code)); + } +#endif + + int index = choice_one(check,cc_max); +#if TEST && GC_DISP + fprintf(stderr,"<choice>\n [%d/0..%d]\n",index,cc_max-1); + fprintf(stderr,"<selected>\n"); + fprintf(stderr," %s\n",charcodeToStr(check[index].code)); +#endif + switch(check[index].code) + { + case cc_unknown: return new_CC_UNKNOWN(); + case cc_ascii: return new_CC_ASCII(); + case cc_sjis: return new_CC_SJIS(); + case cc_eucjp: return new_CC_EUCJP(); + case cc_jis: return new_CC_JIS(); + case cc_utf8: return new_CC_UTF8(); + //case cc_utf32: return new_CC_UTF32(); + case cc_utf32_be: return new_CC_UTF32_BE(); + case cc_utf32_le: return new_CC_UTF32_LE(); + case cc_sjis_jsky: return new_CC_SJIS_JSKY(); + case cc_sjis_imode: return new_CC_SJIS_IMODE(); + case cc_sjis_doti: return new_CC_SJIS_DOTI(); + + default: +#ifdef TEST + return NULL; +#else + return new_CC_UNKNOWN(); +#endif + } + +} + diff --git a/cpan/dist/Unicode-Japanese/getcode.h b/cpan/dist/Unicode-Japanese/getcode.h new file mode 100644 index 00000000..5f8f2738 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/getcode.h @@ -0,0 +1,32 @@ + +#ifndef GETCODE_H +#define GETCODE_H + +// $Id: getcode.h,v 1.1 2001/11/30 13:46:26 hio Exp $ + +#ifdef TEST +#define DECL_MAP_MODE(name,num) \ + extern const char* mode_##name[num] +#else +#define DECL_MAP_MODE(name,num) +#endif + +#define DECL_MAP_TABLE(name,num) \ + extern const unsigned char map_##name[num][256] + +#define DECL_MAP(name,num) DECL_MAP_MODE(name,num); DECL_MAP_TABLE(name,num) + +DECL_MAP(ascii,1); +DECL_MAP(eucjp,5); +DECL_MAP(sjis,2); +DECL_MAP(utf8,6); +DECL_MAP(jis,10); +DECL_MAP(utf32_be,4); +DECL_MAP(utf32_le,4); +DECL_MAP(sjis_jsky,5); +DECL_MAP(sjis_imode,4); +DECL_MAP(sjis_doti,7); + +#define map_invalid 0x7f + +#endif diff --git a/cpan/dist/Unicode-Japanese/getcode_map.cpp b/cpan/dist/Unicode-Japanese/getcode_map.cpp new file mode 100644 index 00000000..bb69d52c --- /dev/null +++ b/cpan/dist/Unicode-Japanese/getcode_map.cpp @@ -0,0 +1,961 @@ + +// $Id: getcode_map.cpp,v 1.5 2002/07/01 00:17:02 hio Exp $ + +#include "getcode.h" + +// << ascii >> +// IVNALID = 127 +// VALID = 0 +extern const unsigned char map_ascii[1][256] = +{ + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [8] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [9] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [a] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [b] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [c] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [d] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [e] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [f] + }, +}; + +// << eucjp >> +// IVNALID = 127 +// VALID = 0 +// ASCII = 0 +// EUCJP:0212 = 1..2 +// EUCJP:C = 3..3 +// EUCJP:KANA = 4..4 +extern const unsigned char map_eucjp[5][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127, 4, 1, // [8] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [9] + 127, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // [a] + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // [b] + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // [c] + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // [d] + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // [e] + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,127, // [f] + }, + { // [1] 0212:3:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [4] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [5] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [6] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [7] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [8] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [9] + 127, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // [a] + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // [b] + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // [c] + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // [d] + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // [e] + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,127, // [f] + }, + { // [2] 0212:3 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [4] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [5] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [6] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [7] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [8] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [f] + }, + { // [3] c:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [4] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [5] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [6] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [7] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [8] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [f] + }, + { // [4] kana:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [4] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [5] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [6] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [7] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [8] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [e] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [f] + }, +}; + +// << sjis >> +// IVNALID = 127 +// VALID = 0 +// ASCII = 0 +// SJIS:DBCS = 1..1 +// SJIS:KANA = 0 +extern const unsigned char map_sjis[2][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [e] + 127,127,127,127,127,127,127,127,127,127, 1, 1, 1,127,127,127, // [f] + }, + { // [1] c:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, +}; + +// << utf8 >> +// IVNALID = 127 +// VALID = 0 +// ASCII = 0 +// UTF8:2 = 5 +// UTF8:3 = 4 +// UTF8:4 = 3 +// UTF8:5 = 2 +// UTF8:6 = 1..5 +extern const unsigned char map_utf8[6][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1,127,127, + }, + { // [1] utf8:6:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [2] utf8:6:3 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [3] utf8:6:4 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [4] utf8:6:5 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [5] utf8:6:6 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, +}; + +extern const unsigned char map_jis[10][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [1] jis_0208(1978,1983):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127, 2,127, 3,127, 7,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [2] jis#2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 8,127, 8,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [3] jis#3 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 4,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [4] jis#4 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127, 5,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [5] jis#5 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127, 6,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [6] jis#6 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127, 8,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [7] jis#7 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127, 0,127,127,127,127,127,127, 8, 0,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [8] jis#loop1 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127, 1,127,127,127,127, + 127, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,127, + }, + { // [9] jis#loop2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,127, + }, +}; + +// << utf32-be >> +// IVNALID = 127 +// VALID = 0 +// UTF32-BE = 1..3 +extern const unsigned char map_utf32_be[4][256] = +{ + { // [0] first + 1,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [1] utf32-be:4:2 + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [2] utf32-be:4:3 + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + }, + { // [3] utf32-be:4 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, +}; + +// << utf32-le >> +// IVNALID = 127 +// VALID = 0 +// UTF32-LE = 1..3 +extern const unsigned char map_utf32_le[4][256] = +{ + { // [0] first + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + }, + { // [1] utf32-le:4:2 + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + }, + { // [2] utf32-le:4:3 + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, + { // [ 3,] utf32-le:4 + 0,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, + }, +}; + +// << sjis >> +// IVNALID = 127 +// VALID = 0 +// ASCII = 0 +// SJIS:C = 1..1 +// SJIS:KANA = 0 +extern const unsigned char map_sjis_jsky[5][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [e] + 127,127,127,127,127,127,127,127,127,127, 1, 1, 1,127,127,127, // [f] + }, + { // [1] c:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [2] jsy-start:2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,127, // [e] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [f] + }, + { // [3] jsky-esc:1 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [1] + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // [2] + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, // [3] + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // [4] + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // [5] + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // [6] + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,127, // [e] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [f] + }, + { // [4] jsky-esc:2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,127, // [e] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [f] + }, +}; + +// << sjis-imode >> +// IVNALID = 127 +// VALID = 0 +// ASCII = 0 +// SJIS:C = 1..1 +// SJIS:KANA = 0 +// emoji:F8xx = 2 +// emoji:F9xx = 3 +extern const unsigned char map_sjis_imode[4][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [e] + 127,127,127,127,127,127,127,127, 2, 3, 1, 1, 1,127, // [f] + }, + { // [1] c:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [ 2,] imode-emoji(1):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [ 3,] imode-emoji(2):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [b] + }, +}; + +// << sjis-doti >> +// IVNALID = 127 +// VALID = 0 +// ASCII = 0 +// SJIS:C = 1..1 +// SJIS:KANA = 0 +extern const unsigned char map_sjis_doti[7][256] = +{ + { // [0] first + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [0] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, 0, 0, 0, 0, // [1] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [2] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [7] + 127, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [8] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [9] + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // [e] + 2, 3, 4, 5, 6,127,127,127,127,127, 1, 1, 1,127,127,127, // [f] + }, + { // [1] c:2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [2] doti-emoji(1):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [3] doti-emoji(2):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0,127,127,127,127,127,127,127,127,127, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [4] doti-emoji(3):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [5] doti-emoji(4):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127,127,127, // [f] + }, + { // [6] doti-emoji(5):2 + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [0] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [1] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [2] + 127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127, // [3] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [4] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [5] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [6] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [7] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [8] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [9] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [a] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [b] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [c] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [d] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // [e] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,127, // [f] + }, +}; + diff --git a/cpan/dist/Unicode-Japanese/jis.cpp b/cpan/dist/Unicode-Japanese/jis.cpp new file mode 100644 index 00000000..e6a5e254 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/jis.cpp @@ -0,0 +1,280 @@ + +// $Id: jis.cpp,v 1.3 2002/01/09 09:13:58 hio Exp $ + +#include "Japanese.h" +#include "sjis.h" + +#define S2J_DISP 0 +#define J2S_DISP 0 + +#define JIS_0208 ((const unsigned char*)"\x1b$B") +#define JIS_0212 ((const unsigned char*)"\x1b$(D") +#define JIS_ASC ((const unsigned char*)"\x1b(B") +#define JIS_KANA ((const unsigned char*)"\x1b(I") +#define JIS_0208_LEN 3 +#define JIS_0212_LEN 4 +#define JIS_ASC_LEN 3 +#define JIS_KANA_LEN 3 + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// sjis=>jis変換 +EXTERN_C +SV* +xs_sjis_jis(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)sjis_jis\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len+8); + int esc_asc = 1; + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + switch(chk_sjis[*src]) + { + case CHK_SJIS_THROUGH: + { // SJIS:THROUGH => JIS:ASCII + if( !esc_asc ) + { + result.append(JIS_ASC,JIS_ASC_LEN); + esc_asc = 1; + } +#if TEST && S2J_DISP + fprintf(stderr," (throuh) %c[%02x]",*src,*src); + fflush(stderr); +#endif + const unsigned char* begin = src; + while( ++src<src_end && chk_sjis[*src]==CHK_SJIS_THROUGH ) + { +#if TEST && S2J_DISP + fprintf(stderr," %c[%02x]",*src,*src); + fflush(stderr); +#endif + } +#if TEST && S2J_DISP + fprintf(stderr,"\n"); + fflush(stderr); +#endif + result.append(begin,src-begin); + break; + } + case CHK_SJIS_C: + { + result.append(JIS_0208,JIS_0208_LEN); + esc_asc = 0; +#if TEST && S2J_DISP + fprintf(stderr," (sjis:c)"); + fflush(stderr); +#endif + do + { +#if TEST && S2J_DISP + fprintf(stderr, "%c%c[%02x.%02x]",src[0],src[1],src[0],src[1]); + fflush(stderr); +#endif + if( src[1]<0x40 || 0xfc<src[1] || src[1]==0x7f ) + { +#if TEST && S2J_DISP + fprintf(stderr, "*"); + fflush(stderr); +#endif + result.append(UNDEF_JIS,UNDEF_JIS_LEN); + ++src; + break; + } + unsigned char tmp[2]; + if( 0x9f <= src[1] ) + { + tmp[0] = src[0]*2 - (src[0]>=0xe0 ? 0xe0 : 0x60); + tmp[1] = src[1] + 2; + }else + { + tmp[0] = src[0]*2 - (src[0]>=0xe0 ? 0xe1 : 0x61); + tmp[1] = src[1] + 0x60 + (src[1] < 0x7f); + } + tmp[0] &= 0x7f; + tmp[1] &= 0x7f; + result.append(tmp,2); + src += 2; + }while( src<src_end && chk_sjis[*src]==CHK_SJIS_C ); +#if TEST && S2J_DISP + fprintf(stderr,"\n"); +#endif + break; + } + case CHK_SJIS_KANA: + { // SJIS:KANA => JIS:KANA + result.append(JIS_KANA,JIS_KANA_LEN); + esc_asc = 0; +#if TEST && S2J_DISP + fprintf(stderr," (sjis:kana)"); + fflush(stderr); +#endif + esc_asc = 0; + do + { +#if TEST && S2J_DISP + fprintf(stderr," %02x",*src); + fflush(stderr); +#endif + result.append(*src&0x7f); + }while( ++src<src_end && chk_sjis[*src]==CHK_SJIS_KANA ); +#if TEST && S2J_DISP + fprintf(stderr,"\n"); +#endif + break; + } + default: + { +#ifdef TEST + fprintf(stderr,"xs_sjis_eucjp, unknown check-code[%02x] on char-code[%05x]\n",chk_sjis[*src],*src); +#endif + result.append(*src++); + } + } //switch + } //while + + if( !esc_asc ) + { + result.append(JIS_ASC,JIS_ASC_LEN); + } + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// jis=>sjis変換 +EXTERN_C +SV* +xs_jis_sjis(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)jis_sjis\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len); + const unsigned char* src_end = src+len; + + if( *src!='\x1b' ) + { + const unsigned char* begin = src; + while( ++src<src_end && *src!='\x1b') + { + } + result.append(begin,src-begin); + } + while( src<src_end ) + { +#if TEST && J2S_DISP + fprintf(stderr," len: %d\n",src_end-src); +#endif + if( src_end-src>=JIS_ASC_LEN && memcmp(src,JIS_ASC,JIS_ASC_LEN)==0 ) + { // <<jis.asc>> + //fprintf(stderr," <jis.asc>\n"); + src += JIS_ASC_LEN; + const unsigned char* begin = src; + while( src<src_end && *src!='\x1b') + { + ++src; + } + if( src!=begin ) + { + result.append(begin,src-begin); + } + }else if( src_end-src>=JIS_0212_LEN && memcmp(src,JIS_0212,JIS_0212_LEN)==0 ) + { // <<jis.0212>> + //fprintf(stderr," <jis.0212>\n"); + src += JIS_0212_LEN; + const unsigned char* begin = src; + while( src<src_end && *src!='\x1b') + { + ++src; + } + for(int i=0; i<(src-begin)/2; ++i ) + { + result.append(UNDEF_SJIS,UNDEF_SJIS_LEN); + } + }else if( src_end-src>=JIS_KANA_LEN && memcmp(src,JIS_KANA,JIS_KANA_LEN)==0 ) + { // <<jis.kana>> + //fprintf(stderr," <jis.kana>\n"); + src += JIS_KANA_LEN; + while( src<src_end && *src!='\x1b') + { + result.append(*src|0x80); + ++src; + } + }else if( src_end-src>=JIS_0208_LEN && memcmp(src,JIS_0208,JIS_0208_LEN)==0 ) + { // <<jis.0208>> +#if TEST && J2S_DISP + fprintf(stderr," <jis.c>"),fflush(stderr); +#endif + src += JIS_0208_LEN; + while( src<src_end ) + { + if( *src=='\x1b' ) break; +#if TEST && J2S_DISP + fprintf(stderr," %02x",src[0]),fflush(stderr); +#endif + if( src+1==src_end || src[1]=='\x1b' ) + { +#if TEST && J2S_DISP + fprintf(stderr,"*"),fflush(stderr); +#endif + ++src; + result.append(UNDEF_SJIS,UNDEF_SJIS_LEN); + break; + } +#if TEST && J2S_DISP + fprintf(stderr," %02x",src[0]),fflush(stderr); +#endif + unsigned char tmp[2]; + tmp[0] = src[0] | 0x80; + tmp[1] = src[1] | 0x80; + if( src[0]%2 ) + { + tmp[0] = (tmp[0]>>1) + (tmp[0] < 0xdf ? 0x31 : 0x71); + tmp[1] = tmp[1] - ( 0x60 + (tmp[1] < 0xe0) ); + }else + { + tmp[0] = (tmp[0]>>1) + (tmp[0] < 0xdf ? 0x30 : 0x70); + tmp[1] = tmp[1] - 2; + } + result.append(tmp,2); + src += 2; + } +#if TEST && J2S_DISP + fprintf(stderr,"\n"); +#endif + }else + { // <<jis.???>> +#ifdef TEST + fprintf(stderr,"xs_jis_sjis, unknown escape found\n"); +#if J2S_DISP + fprintf(stderr," len: %d\n src: %02x %02x %02x\n",src_end-src,src[0],src[1],src[2]); +#endif +#endif + result.append(*src++); + } + } //while + + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} diff --git a/cpan/dist/Unicode-Japanese/memmap.cpp b/cpan/dist/Unicode-Japanese/memmap.cpp new file mode 100644 index 00000000..2883343b --- /dev/null +++ b/cpan/dist/Unicode-Japanese/memmap.cpp @@ -0,0 +1,202 @@ + +// $Id: memmap.cpp,v 1.9 2002/06/25 16:30:27 hio Exp $ + +#include "Japanese.h" +#include <unistd.h> // memmap +#include <sys/mman.h> // memmap +#include <sys/stat.h> // stat +#include <fcntl.h> // open + +#ifndef MAP_FAILED +#define MAP_FAILED ((void*)-1) +#endif + +EXTERN_C +{ + /* SJIS <=> UTF8 変換テーブル */ + unsigned short const* g_u2s_table; + unsigned long const* g_s2u_table; + + /* i-mode/j-sky/dot-i絵文字 <=> UTF8 変換テーブル */ + unsigned long const* g_ei2u_table; + unsigned short const* g_eu2i_table; + unsigned long const* g_ej2u_table; + unsigned char const* g_eu2j_table; // char [][5] + unsigned long const* g_ed2u_table; + unsigned short const* g_eu2d_table; + + /* i-mode/j-sky/dot-i絵文字 <=> UTF8 変換テーブルの要素数 */ + /* バイト数でなく要素数 */ + int g_ei2u_size; + int g_eu2i_size; + int g_ej2u_size; + int g_eu2j_size; + int g_ed2u_size; + int g_eu2d_size; + + /* メモリマップの情報 */ + static int g_mmap_u2s_length; + static char* g_mmap_u2s_start; + static int g_mmap_emj_length; + static char* g_mmap_emj_start; + +} + +static int +findfile(AV* INC, const char* filename) +{ + //fprintf(stderr,"findfile [%s]\n",filename); + int i; + char path[PATH_MAX]; + int addlen = strlen(filename); + for( i=0; i<av_len(INC); ++i ) + { + SV** dir = av_fetch(INC,i,0); + int len = sv_len(*dir); + if( len==0 ) continue; + if( len+addlen+2>=PATH_MAX ) continue; + memcpy(path,SvPV(*dir,PL_na),len); + if( path[len]!='/' ) path[len++] = '/'; + memcpy(path+len,filename,addlen); + path[len+addlen] = '\0'; + //fprintf(stderr," trying [%s] ...\n",path); + int fd = open(path,O_RDONLY|O_NONBLOCK); + if( fd!=-1 ) + { + //fprintf(stderr,"findfile [%s] found\n",filename); + return fd; + } + } + //fprintf(stderr,"findfile [%s] failed\n",filename); + return -1; +} + +void +do_memmap() +{ + //fprintf(stderr,"* Unicode::Japanese::(xs)do_memmap *\n"); + AV* INC = get_av("INC",0); + if( INC==NULL ) + { + Perl_croak(aTHX_ "do_memmap, cannot get @INC."); + return; + } + int fd_u2s = findfile(INC,"Unicode/Japanese/u2s-s2u.dat"); + int fd_emj = findfile(INC,"Unicode/Japanese/emoji.dat"); + + if( fd_u2s==-1 || fd_emj==-1 ) + { + if( fd_u2s!=-1 ) + { + Perl_croak(aTHX_ "do_memmap, emoji table not found or could not open."); + }else if( fd_emj!=-1 ) + { + Perl_croak(aTHX_ "do_memmap, u2s table not found or could not open."); + }else + { + Perl_croak(aTHX_ "do_memmap, u2s table and emoji table not found or could not open."); + } + return; + } + struct stat st_u2s,st_emj; + int res_u2s = fstat(fd_u2s,&st_u2s); + int res_emj = fstat(fd_emj,&st_emj); + if( res_u2s==-1 && res_emj==-1 ) + { + if( res_u2s!=-1 ) + { + Perl_croak(aTHX_ "do_memmap, stat emoji table (for mmap) failed."); + }else if( res_emj!=-1 ) + { + Perl_croak(aTHX_ "do_memmap, stat u2s table (for mmap) failed."); + }else + { + Perl_croak(aTHX_ "do_memmap, stat u2s and emoji tables (for mmap) failed."); + } + close(fd_u2s); + close(fd_emj); + return; + } + + if( st_u2s.st_size!=0x60000 ) + { + Perl_croak(aTHX_ "do_memmap, u2s-s2u size != 0x60000, [got %#x].",st_u2s.st_size); + return; + } + if( st_emj.st_size!=0xb800 ) + { + Perl_croak(aTHX_ "do_memmap, emoji.dat size != 0xb800, [got %#x].",st_emj.st_size); + return; + } + + g_mmap_u2s_length = st_u2s.st_size; + g_mmap_u2s_start = (char*)mmap(NULL,g_mmap_u2s_length,PROT_READ,MAP_PRIVATE,fd_u2s,0); + g_mmap_emj_length = st_emj.st_size; + g_mmap_emj_start = (char*)mmap(NULL,g_mmap_emj_length,PROT_READ,MAP_PRIVATE,fd_emj,0); + + close(fd_u2s); + close(fd_emj); + + if( g_mmap_u2s_start==MAP_FAILED || g_mmap_emj_start==MAP_FAILED ) + { + if( g_mmap_u2s_start!=MAP_FAILED ) + { + Perl_croak(aTHX_ "do_memmap, mmap emoji table failed."); + g_mmap_emj_start = NULL; + }else if( g_mmap_emj_start!=MAP_FAILED ) + { + Perl_croak(aTHX_ "do_memmap, mmap u2s table failed."); + g_mmap_u2s_start = NULL; + }else + { + Perl_croak(aTHX_ "do_memmap, mmap u2s and emoji table failed."); + g_mmap_u2s_start = NULL; + g_mmap_emj_start = NULL; + } + do_memunmap(); + return; + } + + // u2s,s2uの設定 + g_u2s_table = (unsigned short*)(g_mmap_u2s_start + 0x0); + g_s2u_table = (unsigned long *)(g_mmap_u2s_start + 0x20000); + + // i-mode + g_eu2i_table = (unsigned short*)(g_mmap_emj_start + 0x0); // +0x2000 + g_eu2i_size = 0x2000/2; + g_ei2u_table = (unsigned long *)(g_mmap_emj_start + 0x2000); // +0x800 + g_ei2u_size = 0x800/4; + g_eu2j_table = (unsigned char *)(g_mmap_emj_start + 0x2800); // +0x5000 + g_eu2j_size = 0x5000/1; + g_ej2u_table = (unsigned long *)(g_mmap_emj_start + 0x7800); // +0xc00 + g_ej2u_size = 0xc00/4; + g_eu2d_table = (unsigned short*)(g_mmap_emj_start + 0x8400); // +0x2000 + g_eu2d_size = 0x2000/2; + g_ed2u_table = (unsigned long *)(g_mmap_emj_start + 0xa400); // +0x1400 + g_ed2u_size = 0x1400/4; +} + +void +do_memunmap() +{ + // printf("* do_memunmap() *\n"); + + // u2s table + if( g_mmap_u2s_start!=NULL ) + { + int res = munmap(g_mmap_u2s_start,g_mmap_u2s_length); + if( res==-1 ) + { + Perl_croak(aTHX_ "do_memunmap, munmap u2s table failed."); + } + } + // emoji table + if( g_mmap_emj_start!=NULL ) + { + int res = munmap(g_mmap_emj_start,g_mmap_emj_length); + if( res==-1 ) + { + Perl_croak(aTHX_ "do_memunmap, munmap emoji table failed."); + } + } +} diff --git a/cpan/dist/Unicode-Japanese/sjis.cpp b/cpan/dist/Unicode-Japanese/sjis.cpp new file mode 100644 index 00000000..49ddaa59 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/sjis.cpp @@ -0,0 +1,24 @@ + +// $Id: sjis.cpp,v 1.1 2001/12/14 12:59:25 hio Exp $ + +#include "sjis.h" + +extern const unsigned char chk_sjis[256] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 3 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 5 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 6 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 7 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // a + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // b + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // c + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // d + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // e + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // f +}; diff --git a/cpan/dist/Unicode-Japanese/sjis.h b/cpan/dist/Unicode-Japanese/sjis.h new file mode 100644 index 00000000..0449af79 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/sjis.h @@ -0,0 +1,20 @@ + +#ifndef SJIS_H__ +#define SJIS_H__ + +// $Id: sjis.h,v 1.1 2001/12/14 12:59:25 hio Exp $ + +// 変換ができなかったときの文字 +#define UNDEF_SJIS ((const unsigned char*)"\x81\xac") +#define UNDEF_SJIS_LEN 2 +#define UNDEF_JIS ((const unsigned char*)"\xa2\xf7") +#define UNDEF_JIS_LEN 2 + +// sjis=>eucjp変換文字判定 +// 1:SJIS:C, 2:SJIS:KANA +#define CHK_SJIS_THROUGH 0 +#define CHK_SJIS_C 1 +#define CHK_SJIS_KANA 2 +extern const unsigned char chk_sjis[256]; + +#endif diff --git a/cpan/dist/Unicode-Japanese/sjis_doti.cpp b/cpan/dist/Unicode-Japanese/sjis_doti.cpp new file mode 100644 index 00000000..4f0ff9b8 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/sjis_doti.cpp @@ -0,0 +1,296 @@ + +/* $Id: sjis_doti.cpp,v 1.15 2002/07/08 07:50:28 hio Exp $ */ + +#include <stdio.h> +#include "Japanese.h" + +EXTERN_C +SV* +xs_sjis_doti_utf8(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)sjis_doti_utf8\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len*3/2+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + const unsigned char* ptr; + if( src[0]<0x80 ) + { // ASCII + //fprintf(stderr,"ascii: %02x\n",src[0]); + if( src[0]=='&' && src+3<src_end && src[1]=='#' ) + { // "&#ooooo;"のチェック + int num = 0; + unsigned char* ptr = src+2; + const unsigned char* ptr_end = ptr+8<src_end ? ptr+8 : src_end; + for( ; ptr<ptr_end; ++ptr ) + { + if( *ptr==';' ) break; + if( *ptr<'0' || '9'<*ptr ) break; + num = num*10 + *ptr-'0'; + } + if( ptr<ptr_end && *ptr==';' && 0xf000<=num && num<=0xf4ff ) + { // &#oooo;表記のdot-i絵文字 + const unsigned char* emoji = (unsigned char*)&g_ed2u_table[num-0xf000]; + if( emoji[3] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",4); + result.append(emoji,4); + src = ptr+1; + continue; + } + } + } + + result.append(*src++); + continue; + }else if( 0xa1<=src[0] && src[0]<=0xdf ) + { // 半角カナ + //fprintf(stderr,"kana": %02x\n",src[0]); + ptr = (unsigned char*)&g_s2u_table[src[0]]; + ++src; + }else if( src+1<src_end && ( 0xf0<=src[0] && src[0]<=0xf4 ) ) + { // dot-i絵文字 + ptr = (unsigned char*)&g_ed2u_table[((src[0]&0x07)<<8)|src[1]]; + if( *(unsigned long*)ptr==0 ) + { + register const unsigned short sjis = ntohs(*(unsigned short*)src); + //fprintf(stderr,"sjis: %04x\n",sjis); + ptr = (unsigned char*)&g_s2u_table[sjis]; + } + src += 2; + }else if( ((0x81<=src[0] && src[0]<=0x9f) || (0xe0<=src[0] && src[0]<=0xfc) ) + && src+1<src_end + && (0x40<=src[1] && src[1]<=0xfc && src[1]!=0x7f) ) + { // 2バイト文字 + register const unsigned short sjis = ntohs(*(unsigned short*)src); + //fprintf(stderr,"sjis: %04x\n",sjis); + ptr = (unsigned char*)&g_s2u_table[sjis]; + src += 2; + }else + { // 不明 + //fprintf(stderr,"unknown: %02x\n",src[0]); + result.append('?'); + ++src; + continue; + } + + //fprintf(stderr,"utf8-char : %02x %02x %02x %02x\n",ptr[0],ptr[1],ptr[2],ptr[3]); + if( ptr[3] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",4); + result.append_ch4(*(int*)ptr); + }else if( ptr[2] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",3); + result.append_ch3(*(int*)ptr); + }else if( ptr[1] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",2); + result.append_ch2(*(short*)ptr); + }else + { + //fprintf(stderr,"utf8-len: [%d]\n",1); + result.append(*ptr); + } + } + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} + +EXTERN_C +SV* +xs_utf8_sjis_doti(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)utf8_sjis_doti\n"); + //ON_U2S( bin_dump("in ",src,len) ); + + SV_Buf result(len+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + if( *src<=0x7f ) + { + // ASCIIはまとめて追加〜 + int len = 1; + while( src+len<src_end && src[len]<=0x7f ) + { + ++len; + } + result.append(src,len); + src+=len; + continue; + } + // utf8をucsに変換 + // utf8の1文字の長さチェック + int utf8_len; + if( 0xc0<=*src && *src<=0xdf ) + { + utf8_len = 2; + }else if( 0xe0<=*src && *src<=0xef ) + { + utf8_len = 3; + }else if( 0xf0<=*src && *src<=0xf7 ) + { + utf8_len = 4; + }else if( 0xf8<=*src && *src<=0xfb ) + { + utf8_len = 5; + }else if( 0xfc<=*src && *src<=0xfd ) + { + utf8_len = 6; + }else + { + result.append('?'); + ++src; + continue; + } + // 長さ足りてるかチェック + if( src+utf8_len-1>=src_end ) + { + //ECHO_U2S((stderr," no enough buffer, here is %d, need %d\n",src_end-src,utf8_len)); + result.append('?'); + ++src; + continue; + } + // 2バイト目以降が正しい文字範囲か確認 + bool succ = true; + for( int i=1; i<utf8_len; ++i ) + { + if( src[i]<0x80 || 0xbf<src[i] ) + { + //ECHO_U2S((stderr," at %d, char out of range\n",i)); + succ = false; + break; + } + } + if( !succ ) + { + result.append('?'); + ++src; + continue; + } + // utf8からucsのコードを算出 + //ECHO_U2S((stderr,"utf8-charlen: [%d]\n",utf8_len)); + unsigned int ucs; + switch(utf8_len) + { + case 2: + { + ucs = ((src[0] & 0x1F)<<6)|(src[1] & 0x3F); + break; + } + case 3: + { + ucs = ((src[0] & 0x0F)<<12)|((src[1] & 0x3F)<<6)|(src[2] & 0x3F); + break; + } + case 4: + { + ucs = ((src[0] & 0x07)<<18)|((src[1] & 0x3F)<<12)| + ((src[2] & 0x3f) << 6)|(src[3] & 0x3F); + break; + } + case 5: + { + ucs = ((src[0] & 0x03) << 24)|((src[1] & 0x3F) << 18)| + ((src[2] & 0x3f) << 12)|((src[3] & 0x3f) << 6)| + (src[4] & 0x3F); + break; + } + case 6: + { + ucs = ((src[0] & 0x03) << 30)|((src[1] & 0x3F) << 24)| + ((src[2] & 0x3f) << 18)|((src[3] & 0x3f) << 12)| + ((src[4] & 0x3f) << 6)|(src[5] & 0x3F); + break; + } + default: + { + // NOT REACH HERE + //ECHO_U2S((stderr,"invalid utf8-length: %d\n",utf8_len)); + ucs = '?'; + } + } + + if( 0x0f0000<=ucs && ucs<=0x0fffff ) + { // 私用領域 + assert(utf8_len>=4); + if( ucs<0x0ff000 ) + { // 知らない使用領域 + result.append('?'); + src += utf8_len; + continue; + } + // 絵文字判定(dot-i) + const unsigned char* const sjis = (unsigned char*)&g_eu2d_table[ucs - 0x0ff000]; + if( sjis[1]!=0 ) + { // 2バイト文字に. + result.append_ch2(*reinterpret_cast<const unsigned short*>(sjis)); + }else if( sjis[0]!=0 ) + { // 1バイト文字に. + result.append(*sjis); + }else + { // マッピングなし + result.append('?'); + } + src += utf8_len; + continue; + } + + if( ucs & ~0xFFFF ) + { // ucs2の範囲外 (ucs4の範囲) + result.append('?'); + src += utf8_len; + continue; + } + + // ucs => sjis + //ECHO_U2S((stderr,"ucs2 [%04x]\n",ucs)); + const unsigned short sjis = g_u2s_table[ucs]; + //ECHO_U2S((stderr,"sjis [%04x]\n",ntohs(sjis) )); + + if( sjis || !ucs ) + { // 対応文字がある時とucs=='\0'の時 + if( sjis & 0xff00 ) + { + result.append_ch2(sjis); + }else + { + result.append((unsigned char)sjis); + } + }else if( ucs<=0x7F ) + { + result.append((unsigned char)ucs); + }else + { + result.append('?'); + } + src += utf8_len; + //bin_dump("now",dst_begin,dst-dst_begin); + } /* for */ + + //ON_U2S( bin_dump("out",result.getBegin(),result.getLength()) ); + result.setLength(); + + return result.getSv(); +} diff --git a/cpan/dist/Unicode-Japanese/sjis_imode.cpp b/cpan/dist/Unicode-Japanese/sjis_imode.cpp new file mode 100644 index 00000000..2df09a85 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/sjis_imode.cpp @@ -0,0 +1,303 @@ + +// $Id: sjis_imode.cpp,v 1.15 2002/07/04 04:52:29 hio Exp $ + +#include <stdio.h> +#include "Japanese.h" + +EXTERN_C +SV* +xs_sjis_imode_utf8(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)sjis_imode_utf8[len:%d]\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len*3/2+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + const unsigned char* ptr; + if( src[0]<0x80 ) + { // ASCII + //fprintf(stderr,"ascii: %02x\n",src[0]); + if( src[0]=='&' && src+3<src_end && src[1]=='#' ) + { // "&#ooooo;"のチェック + int num = 0; + unsigned char* ptr = src+2; + const unsigned char* ptr_end = ptr+8<src_end ? ptr+8 : src_end; + for( ; ptr<ptr_end; ++ptr ) + { + if( *ptr==';' ) break; + if( *ptr<'0' || '9'<*ptr ) break; + num = num*10 + *ptr-'0'; + } + if( ptr<ptr_end && *ptr==';' && 0xf800<=num && num<=0xf9ff ) + { // &#oooo;表記のi-mode絵文字 + const unsigned char* emoji = (unsigned char*)&g_ei2u_table[num&0x1ff]; + if( emoji[3] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",4); + result.append(emoji,4); + src = ptr+1; + continue; + } + } + } + + result.append(*src++); + continue; + }else if( 0xa1<=src[0] && src[0]<=0xdf ) + { // 半角カナ + //fprintf(stderr,"kana": %02x\n",src[0]); + ptr = (unsigned char*)&g_s2u_table[src[0]]; + ++src; + }else if( src+1<src_end && ( src[0]==0xf8 || src[0]==0xf9 ) ) + { // i-mode絵文字 + //fprintf(stderr,"code: %02x %02x\n", src[0],src[1]); + ptr = (unsigned char*)&g_ei2u_table[((src[0]&1)<<8)|src[1]]; + if( *(unsigned long*)ptr==0 ) + { + register const unsigned short sjis = ntohs(*(unsigned short*)src); + //fprintf(stderr,"sjis: %04x\n",sjis); + ptr = (unsigned char*)&g_s2u_table[sjis]; + } + //fprintf(stderr,"out : %02x %02x %02x %02x\n",ptr[0],ptr[1],ptr[2],ptr[3]); + src += 2; + }else if( ((0x81<=src[0] && src[0]<=0x9f) || (0xe0<=src[0] && src[0]<=0xfc) ) + && src+1<src_end + && (0x40<=src[1] && src[1]<=0xfc && src[1]!=0x7f) ) + { // 2バイト文字 + unsigned short sjis = ntohs(*(unsigned short*)src); + //fprintf(stderr,"sjis: %04x\n",sjis); + ptr = (unsigned char*)&g_s2u_table[sjis]; + src += 2; + }else + { // 不明 + //fprintf(stderr,"unknown: %02x\n",src[0]); + result.append('?'); + ++src; + continue; + } + + //fprintf(stderr,"utf8-char : %02x %02x %02x %02x\n",ptr[0],ptr[1],ptr[2],ptr[3]); + if( ptr[3] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",4); + result.append_ch4(*(int*)ptr); + }else if( ptr[2] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",3); + result.append_ch3(*(int*)ptr); + }else if( ptr[1] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",2); + result.append_ch2(*(short*)ptr); + }else if( ptr[0] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",1); + result.append(*ptr); + }else + { + result.append('?'); + } + } + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} + + +EXTERN_C +SV* +xs_utf8_sjis_imode(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)utf8_sjis_imode\n"); + //ON_U2S( bin_dump("in ",src,len) ); + + SV_Buf result(len+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + if( *src<=0x7f ) + { + // ASCIIはまとめて追加〜 + int len = 1; + while( src+len<src_end && src[len]<=0x7f ) + { + ++len; + } + result.append(src,len); + src+=len; + continue; + } + // utf8をucsに変換 + // utf8の1文字の長さチェック + int utf8_len; + if( 0xc0<=*src && *src<=0xdf ) + { + utf8_len = 2; + }else if( 0xe0<=*src && *src<=0xef ) + { + utf8_len = 3; + }else if( 0xf0<=*src && *src<=0xf7 ) + { + utf8_len = 4; + }else if( 0xf8<=*src && *src<=0xfb ) + { + utf8_len = 5; + }else if( 0xfc<=*src && *src<=0xfd ) + { + utf8_len = 6; + }else + { + result.append('?'); + ++src; + continue; + } + // 長さ足りてるかチェック + if( src+utf8_len-1>=src_end ) + { + //ECHO_U2S((stderr," no enough buffer, here is %d, need %d\n",src_end-src,utf8_len)); + result.append('?'); + ++src; + continue; + } + // 2バイト目以降が正しい文字範囲か確認 + bool succ = true; + for( int i=1; i<utf8_len; ++i ) + { + if( src[i]<0x80 || 0xbf<src[i] ) + { + //ECHO_U2S((stderr," at %d, char out of range\n",i)); + succ = false; + break; + } + } + if( !succ ) + { + result.append('?'); + ++src; + continue; + } + // utf8からucsのコードを算出 + //ECHO_U2S((stderr,"utf8-charlen: [%d]\n",utf8_len)); + unsigned int ucs; + switch(utf8_len) + { + case 2: + { + ucs = ((src[0] & 0x1F)<<6)|(src[1] & 0x3F); + break; + } + case 3: + { + ucs = ((src[0] & 0x0F)<<12)|((src[1] & 0x3F)<<6)|(src[2] & 0x3F); + break; + } + case 4: + { + ucs = ((src[0] & 0x07)<<18)|((src[1] & 0x3F)<<12)| + ((src[2] & 0x3f) << 6)|(src[3] & 0x3F); + break; + } + case 5: + { + ucs = ((src[0] & 0x03) << 24)|((src[1] & 0x3F) << 18)| + ((src[2] & 0x3f) << 12)|((src[3] & 0x3f) << 6)| + (src[4] & 0x3F); + break; + } + case 6: + { + ucs = ((src[0] & 0x03) << 30)|((src[1] & 0x3F) << 24)| + ((src[2] & 0x3f) << 18)|((src[3] & 0x3f) << 12)| + ((src[4] & 0x3f) << 6)|(src[5] & 0x3F); + break; + } + default: + { + // NOT REACH HERE + //ECHO_U2S((stderr,"invalid utf8-length: %d\n",utf8_len)); + ucs = '?'; + } + } + + if( 0x0f0000<=ucs && ucs<=0x0fffff ) + { // 私用領域 + assert(utf8_len>=4); + if( ucs<0x0ff000 ) + { // 知らない使用領域 + result.append('?'); + src += utf8_len; + continue; + } + // 絵文字判定(imode) + const unsigned char* const sjis = (unsigned char*)&g_eu2i_table[ucs - 0x0ff000]; + if( sjis[1]!=0 ) + { // 2バイト文字に. + result.append_ch2(*reinterpret_cast<const unsigned short*>(sjis)); + }else if( sjis[0]!=0 ) + { // 1バイト文字に. + result.append(*sjis); + }else + { // マッピングなし + result.append('?'); + } + src += utf8_len; + continue; + } + + if( ucs & ~0xFFFF ) + { // ucs2の範囲外 (ucs4の範囲) + result.append('?'); + src += utf8_len; + continue; + } + + // ucs => sjis + //ECHO_U2S((stderr,"ucs2 [%04x]\n",ucs)); + const unsigned short sjis = g_u2s_table[ucs]; + //ECHO_U2S((stderr,"sjis [%04x]\n",ntohs(sjis) )); + + if( sjis || !ucs ) + { // 対応文字がある時とucs=='\0'の時 + if( sjis & 0xff00 ) + { + result.append_ch2(sjis); + }else + { + result.append((unsigned char)sjis); + } + }else if( ucs<=0x7F ) + { + result.append((unsigned char)ucs); + }else + { + result.append('?'); + } + src += utf8_len; + //bin_dump("now",dst_begin,dst-dst_begin); + } /* for */ + + //ON_U2S( bin_dump("out",result.getBegin(),result.getLength()) ); + result.setLength(); + + return result.getSv(); +} + diff --git a/cpan/dist/Unicode-Japanese/sjis_jsky.cpp b/cpan/dist/Unicode-Japanese/sjis_jsky.cpp new file mode 100644 index 00000000..f898a0cf --- /dev/null +++ b/cpan/dist/Unicode-Japanese/sjis_jsky.cpp @@ -0,0 +1,377 @@ + +// $Id: sjis_jsky.cpp,v 1.18 2002/07/04 04:52:29 hio Exp $ + +#include <stdio.h> +#include "Japanese.h" + +#define ECHO_EJ2U(arg) //fprintf arg +#define ON_EJ2U(cmd) //cmd +#define ECHO_U2EJ(arg) //fprintf arg +#define ON_U2EJ(cmd) //cmd + +EXTERN_C +SV* +xs_sjis_jsky_utf8(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + const unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + ECHO_EJ2U((stderr,"Unicode::Japanese::(xs)sjis_jsky_utf8\n",len)); + ON_EJ2U( bin_dump("in ",src,len) ); + + SV_Buf result(len*3/2+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + const unsigned char* ptr; + if( src[0]<0x80 ) + { // ASCII + //fprintf(stderr,"ascii: %02x\n",src[0]); + if( src[0]!='\e' || src+2>=src_end || src[1]!='$' ) + { // 絵文字じゃない + result.append(*src++); + continue; + } + //fprint(stderr,"detect j-sky emoji-start escape\n"); + // E_JSKY_1 + if( src[2]!='E' && src[2]!='F' && src[2]!='G' ) + { + //fprintf(stderr,"first char is invalid"); + result.append(*src++); + continue; + } + + const unsigned char* begin = src; + src += 3; + // E_JSKY_2 + while( src+1<src_end ) + { + if( '!'<=src[0] && src[0]<='z' ) + { + ++src; + continue; + } + break; + } + if( src[0]!=0x0f ) + { + //fprintf(stderr,"invalid\n"); + src = begin; + result.append(*src++); + continue; + } + ++src; + const int j1 = (begin[2]-'E')<<8; + for( const unsigned char* ptr = begin+3; ptr<src-1; ++ptr ) + { + //fprintf(stderr," <%c%c:%04x>\n",begin[2],*ptr,j1+*ptr); + //fprintf(stderr," => %04x\n",g_ej2u_table[j1+*ptr]); + const unsigned char* str = (unsigned char*)&g_ej2u_table[j1+*ptr]; + //fprintf(stderr," len: %d\n",str[3]?4:strlen((char*)str)); + result.append(str,str[3]?4:strlen((char*)str)); + } + //fprintf(stderr,"j-sky string done.\n"); + continue; + }else if( 0xa1<=src[0] && src[0]<=0xdf ) + { // 半角カナ + //fprintf(stderr,"kana": %02x\n",src[0]); + ptr = (unsigned char*)&g_s2u_table[src[0]]; + ++src; + }else if( ((0x81<=src[0] && src[0]<=0x9f) || (0xe0<=src[0] && src[0]<=0xfc) ) + && (0x40<=src[1] && src[1]<=0xfc && src[1]!=0x7f) ) + { // 2バイト文字 + register const unsigned short sjis = ntohs(*(unsigned short*)src); + //fprintf(stderr,"sjis: %04x\n",sjis); + ptr = (unsigned char*)&g_s2u_table[sjis]; + src += 2; + }else + { // 不明 + //fprintf(stderr,"unknown: %02x\n",src[0]); + result.append('?'); + ++src; + continue; + } + + //fprintf(stderr,"utf8-char : %02x %02x %02x %02x\n",ptr[0],ptr[1],ptr[2],ptr[3]); + if( ptr[3] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",4); + result.append_ch4(*(int*)ptr); + }else if( ptr[2] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",3); + result.append_ch3(*(int*)ptr); + }else if( ptr[1] ) + { + //fprintf(stderr,"utf8-len: [%d]\n",2); + result.append_ch2(*(short*)ptr); + }else + { + //fprintf(stderr,"utf8-len: [%d]\n",1); + result.append(*ptr); + } + } + ON_EJ2U( bin_dump("out",result.getBegin(),result.getLength()) ); + result.setLength(); + + return result.getSv(); +} + + +EXTERN_C +SV* +xs_utf8_sjis_jsky(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVsv(&PL_sv_undef); + } + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + int len = sv_len(sv_str); + + ECHO_U2EJ((stderr,"Unicode::Japanese::(xs)utf8_sjis\n")); + ON_U2EJ( bin_dump("in ",src,len) ); + + SV_Buf result(len+4); + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + if( *src<=0x7f ) + { + // ASCIIはまとめて追加〜 + int len = 1; + while( src+len<src_end && src[len]<=0x7f ) + { + ++len; + } + result.append(src,len); + src+=len; + continue; + } + // utf8をucsに変換 + // utf8の1文字の長さチェック + int utf8_len; + if( 0xc0<=*src && *src<=0xdf ) + { + utf8_len = 2; + }else if( 0xe0<=*src && *src<=0xef ) + { + utf8_len = 3; + }else if( 0xf0<=*src && *src<=0xf7 ) + { + utf8_len = 4; + }else if( 0xf8<=*src && *src<=0xfb ) + { + utf8_len = 5; + }else if( 0xfc<=*src && *src<=0xfd ) + { + utf8_len = 6; + }else + { + result.append('?'); + ++src; + continue; + } + // 長さ足りてるかチェック + if( src+utf8_len-1>=src_end ) + { + ECHO_U2EJ((stderr," no enough buffer, here is %d, need %d\n",src_end-src,utf8_len)); + result.append('?'); + ++src; + continue; + } + // 2バイト目以降が正しい文字範囲か確認 + bool succ = true; + for( int i=1; i<utf8_len; ++i ) + { + if( src[i]<0x80 || 0xbf<src[i] ) + { + ECHO_U2EJ((stderr," at %d, char out of range\n",i)); + succ = false; + break; + } + } + if( !succ ) + { + result.append('?'); + ++src; + continue; + } + // utf8からucsのコードを算出 + ECHO_U2EJ((stderr,"utf8-charlen: [%d]\n",utf8_len)); + unsigned int ucs; + switch(utf8_len) + { + case 2: + { + ucs = ((src[0] & 0x1F)<<6)|(src[1] & 0x3F); + break; + } + case 3: + { + ucs = ((src[0] & 0x0F)<<12)|((src[1] & 0x3F)<<6)|(src[2] & 0x3F); + break; + } + case 4: + { + ucs = ((src[0] & 0x07)<<18)|((src[1] & 0x3F)<<12)| + ((src[2] & 0x3f) << 6)|(src[3] & 0x3F); + break; + } + case 5: + { + ucs = ((src[0] & 0x03) << 24)|((src[1] & 0x3F) << 18)| + ((src[2] & 0x3f) << 12)|((src[3] & 0x3f) << 6)| + (src[4] & 0x3F); + break; + } + case 6: + { + ucs = ((src[0] & 0x03) << 30)|((src[1] & 0x3F) << 24)| + ((src[2] & 0x3f) << 18)|((src[3] & 0x3f) << 12)| + ((src[4] & 0x3f) << 6)|(src[5] & 0x3F); + break; + } + default: + { + // NOT REACH HERE + ECHO_U2EJ((stderr,"invalid utf8-length: %d\n",utf8_len)); + ucs = '?'; + } + } + + if( 0x0f0000<=ucs && ucs<=0x0fffff ) + { // 私用領域 + assert(utf8_len>=4); + if( ucs<0x0ff000 ) + { // 知らない使用領域 + result.append('?'); + src += utf8_len; + continue; + } + // 絵文字判定(j-sky) + const unsigned char* const sjis = &g_eu2j_table[(ucs - 0x0ff000)*5]; + //fprintf(stderr," emoji: %02x %02x %02x %02x %02x\n", + // sjis[0],sjis[1],sjis[2],sjis[3],sjis[4]); + if( sjis[4]!=0 ) + { // 5バイト文字に. + result.append_ch5(sjis); + }else if( sjis[3]!=0 ) + { // 4バイト文字に. + assert("not reach here" && 0); + result.append_ch4(*reinterpret_cast<const int*>(sjis)); + }else if( sjis[2]!=0 ) + { // 3バイト文字に. + assert("not reach here" && 0); + result.append_ch3(*reinterpret_cast<const int*>(sjis)); + }else if( sjis[1]!=0 ) + { // 2バイト文字に. + result.append_ch2(*reinterpret_cast<const unsigned short*>(sjis)); + }else if( sjis[0]!=0 ) + { // 1バイト文字に. + result.append(*sjis); + }else + { // マッピングなし + result.append('?'); + } + src += utf8_len; + continue; + } + + if( ucs & ~0xFFFF ) + { // ucs2の範囲外 (ucs4の範囲) + result.append('?'); + src += utf8_len; + continue; + } + + // ucs => sjis + ECHO_U2EJ((stderr,"ucs2 [%04x]\n",ucs)); + const unsigned short sjis = g_u2s_table[ucs]; + ECHO_U2EJ((stderr,"sjis [%04x]\n",ntohs(sjis) )); + + if( sjis || !ucs ) + { // 対応文字がある時とucs=='\0'の時 + if( sjis & 0xff00 ) + { + result.append_ch2(sjis); + }else + { + result.append((unsigned char)sjis); + } + }else if( ucs<=0x7F ) + { + result.append((unsigned char)ucs); + }else + { + result.append('?'); + } + src += utf8_len; + //bin_dump("now",dst_begin,dst-dst_begin); + } /* for */ + + ON_U2EJ( bin_dump("out",result.getBegin(),result.getLength()) ); + result.setLength(); + sv_2mortal(result.getSv()); + + // packing J-SKY emoji escapes + SV_Buf pack(result.getLength()); + src = result.getBegin(); + src_end = src + result.getLength(); + unsigned char* ptr = src; + for( ; src+5*2-1<src_end; ++src ) + { + // E_JSKY_START "\e\$", + if( src[0]!='\x1b' ) continue; + if( src[1]!='$' ) continue; + // E_JSKY1 '[EFG]', + //fprintf(stderr," found emoji-start\n"); + if( src[2]!='E' && src[2]!='F' && src[2]!='G' ) + { + //fprintf(stderr," invalid ch1 [%x:%02x]\n",src[2],src[2]); + continue; + } + unsigned char ch1 = src[2]; + // E_JSKY2 '[\!-\;\=-z\xbc]', + if( src[3]<'!' || 'z'<src[3] ) + { + //fprintf(stderr," invalid ch2 [%02x]\n",src[3]); + continue; + } + // E_JSKY_END "\x0f", + if( src[4]!='\x0f' ) continue; + + //fprintf(stderr," found first emoji [%02x:%c]\n",ch1,ch1); + src += 5; + pack.append(ptr,(src-1)-ptr); + unsigned char tmpl[5] = { '\x1b','$',0,0,'\x0f',}; + tmpl[2] = ch1; + for( ; src_end-src>=5; src+= 5 ) + { + tmpl[3] = src[3]; + if( memcmp(src,tmpl,5)!=0 ) break; + //fprintf(stderr," packing...[%02x]\n",src[3]); + pack.append(src[3]); + } + //fprintf(stderr," pack done.\n"); + pack.append('\x0f'); + ptr = src; + } + //fprintf(stderr," pack complete.\n"); + //fprintf(stderr," append len %0d\n",src_end-ptr); + if( ptr!=src_end ) + { + pack.append(ptr,src_end-ptr); + } + + ON_U2EJ( bin_dump("out",pack.getBegin(),pack.getLength()) ); + pack.setLength(); + + return pack.getSv(); +} diff --git a/cpan/dist/Unicode-Japanese/str.h b/cpan/dist/Unicode-Japanese/str.h new file mode 100644 index 00000000..71b6d6e6 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/str.h @@ -0,0 +1,148 @@ + +#ifndef STR_H__ +#define STR_H__ + +// $Id: str.h,v 1.7 2002/07/04 04:52:29 hio Exp $ + +// BUF_MALLOC : use malloc() +// (undef) : use SV* buffer directly +//#define BUF_MALLOC + +class SV_Buf +{ +private: +#ifndef BUF_MALLOC + SV* sv; +#endif + STRLEN alloc_len; + unsigned char* dst; + unsigned char* dst_begin; + +public: +#ifndef BUF_MALLOC + SV_Buf(STRLEN len) : alloc_len(len) + { + sv = newSVpvn("",0); + STRLEN alen = alloc_len+1; + //fprintf(stderr,"sv = %#08x, len = %d\n",sv,alen); + SvGROW(sv,alen); + dst = (unsigned char*)SvPV(sv,alen); + dst_begin = dst; + } +#else + SV_Buf(STRLEN len) : alloc_len(len) + { + dst = (unsigned char*)malloc(alloc_len+1); + dst_begin = dst; + fprintf(stderr,"malloc = %#08x\n",dst_begin); + } + ~SV_Buf() + { + free(dst_begin); + } +#endif + STRLEN getLength(){ return dst-dst_begin; } +#ifndef BUF_MALLOC + void setLength(){ SvCUR_set(sv,dst-dst_begin); } +#else + void setLength(){} +#endif + unsigned char* getBegin(){ return dst_begin; } + SV* getSv() + { +#ifndef BUF_MALLOC + return sv; +#else + return newSVpvn((char*)dst_begin,dst-dst_begin); +#endif + } + inline void append(unsigned char ch) + { // same as append_ch + checkbuf(1); + *dst++ = ch; + } + inline void append_ch(unsigned char ch) + { + checkbuf(1); + *dst++ = ch; + } + inline void append_ch2(unsigned short ch) + { + checkbuf(2); + *(unsigned short*)dst = ch; + dst += 2; + } + inline void append_ch3(int ch) + { + checkbuf(4); + *(int*)dst = ch; + dst += 3; + } + inline void append_ch4(int ch) + { + checkbuf(4); + *(int*)dst = ch; + dst += 4; + } + inline void append_ch5(const unsigned char* src) + { + checkbuf(5); + memcpy(dst,src,5); + dst += 5; + } + inline void append(const unsigned char* src, int len) + { + checkbuf(len); + memcpy(dst,src,len); + dst += len; + } + // entity reference で追加 + inline void append_entityref(unsigned long ucs) + { + char buf[32]; + register int write_len = snprintf(buf,32,"&#%lu;",ucs); + if( write_len!=-1 && write_len<32 ) + { + append((unsigned char*)buf,write_len); + }else + { // 失敗するコトなんてないと思うけど…. + // -1はglibc2.0.6以前, 2.1以降は必要なサイズ + append_ch('?'); + } + } + void checkbuf(STRLEN len) + { +#ifdef TEST + if( len==0 ) + { + fprintf(stderr,"SV_Buf.checkbuf, check length 0.\n"); + } +#endif + if( (dst-dst_begin)+len>=alloc_len ) + { + STRLEN now_len = dst-dst_begin; + STRLEN new_len = (alloc_len+len)*2; +#ifdef TEST + fprintf(stderr,"<<SV_Buf.realloc>> %d+%d/%d => %d\n",now_len,len,alloc_len,new_len); +#endif +#ifndef BUF_MALLOC + setLength(); + STRLEN alen = new_len+1; + SvGROW(sv,alen); + STRLEN curlen; + dst_begin = (unsigned char*)SvPV(sv,curlen); +#else + unsigned char* buf = (unsigned char*)malloc(new_len+1); + memcpy(buf,dst_begin,now_len); + free(dst_begin); + dst_begin = buf; +#endif + alloc_len = new_len; + dst = dst_begin + now_len; + } + } +}; + +#endif + + diff --git a/cpan/dist/Unicode-Japanese/t/allchar_utf8.pl b/cpan/dist/Unicode-Japanese/t/allchar_utf8.pl new file mode 100644 index 00000000..d27081ec --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/allchar_utf8.pl @@ -0,0 +1,65 @@ +#!/usr/bin/perl + +use strict; +use Unicode::Japanese; + +# とりあえず utf8=>* 変換してみるだけ. +# あってるかはみてないにゃ. +# 3文字しかしてなかったり… +# utf8は6文字まで? +# + +our @charcodes = ( + 'jis', 'sjis', 'euc', + 'sjis-imode', 'sjis-doti', 'sjis-jsky', + ); + +# in: utf8 0x00 - 0xFF.FF.FF + +$| = 1; +for( my $i=0; $i<=0xFFFFFF; ++$i ) +{ + if( ($i&0xFF)==0 ) + { + if( ($i&0x3FFF)==0 ) + { + print "\n" if( $i ); + printf "[%#08x]",$i; + }else + { + print "."; + } + } + + my $src = pack('N',$i); + $src =~ s/^\0+//; + + # ------------------------------------ + # utf8 => jis/eucjp/etc. + # + my $str = Unicode::Japanese->new($src,'utf8'); + foreach my $ocode ( @charcodes ) + { + $str->conv($ocode); + } + + # ------------------------------------ + # jis/eucjp/etc. => utf8 + foreach my $icode ( @charcodes ) + { + Unicode::Japanese->new($src,$icode); + } +} + +print "\n"; + +check_mem(); +sub check_mem +{ + open FILE,"/proc/$$/status" or die "cannot open [/proc/$$/status]"; + while(<FILE>) + { + m/^Vm\w+:\s*(\d+)/m and print; + } + close FILE; +} diff --git a/cpan/dist/Unicode-Japanese/t/base.t b/cpan/dist/Unicode-Japanese/t/base.t new file mode 100644 index 00000000..a18b9efc --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/base.t @@ -0,0 +1,27 @@ + +use Test; + +BEGIN { plan tests => 5 } + +require Unicode::Japanese; +ok(1); + +import Unicode::Japanese; +ok(1); + +## check new and set/get + +my $string; + +$string = new Unicode::Japanese; +ok($string); + +$string = new Unicode::Japanese 'abcde'; +ok($string->get, 'abcde'); + +$string = new Unicode::Japanese; +$string->set('abcde'); +ok($string->get, 'abcde'); + + + diff --git a/cpan/dist/Unicode-Japanese/t/emoji.t b/cpan/dist/Unicode-Japanese/t/emoji.t new file mode 100644 index 00000000..18ef9ace --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/emoji.t @@ -0,0 +1,75 @@ +#!/usr/bin/perl + +use Test; +BEGIN{ plan tests => 5*5; } + +use strict; +use Unicode::Japanese; + +print STDERR $Unicode::Japanese::PurePerl?"PurePerl mode\n":"XS mode\n"; + +# test(code, +# ucs4, sjis, imode, jsky, doti ); +# code: imode|jsky|doti +# +# + +# imode絵文字のテスト +# +test( 'imode', # 晴れ + 0x0FF89F, '?',"\xF8\x9F", "\e\$Gj\x0f", "\xF0\xE5" ); +test( 'imode', # 曇り + 0x0FF8A0, '?',"\xF8\xA0", "\e\$Gi\x0f", "\xF0\xE6" ); +test( 'imode', # 雨 + 0x0FF8A1, '?',"\xF8\xA1", "\e\$Gk\x0f", "\xF0\xE7" ); +test( 'imode', # 雪 + 0x0FF8A2, '?',"\xF8\xA2", "\e\$Gh\x0f", "\xF0\xE8" ); +test( 'imode', # 雷 + 0x0FF8A3, '?',"\xF8\xA3", "\e\$E]\x0f", "\xF0\xE9" ); + +sub test +{ + my $code = shift; + my $ucs4 = shift; + my $sjis = shift; + my $imode = shift; + my $jsky = shift; + my $doti = shift; + + $sjis = '&#'.$ucs4.';' unless( defined($sjis) ); + $imode = '&#'.$ucs4.';' unless( defined($imode) ); + $jsky = '&#'.$ucs4.';' unless( defined($jsky) ); + $doti = '&#'.$ucs4.';' unless( defined($doti) ); + $ucs4 = pack('N',$ucs4); + + my $str = $code eq 'imode' ? Unicode::Japanese->new($imode,'sjis-imode') : + $code eq 'jsky' ? Unicode::Japanese->new($imode,'sjis-jsky') : + $code eq 'doti' ? Unicode::Japanese->new($imode,'sjis-doti') : + die "code invalid [$code]"; + + ($ucs4,$sjis,$imode,$jsky,$doti) = escl($ucs4,$sjis,$imode,$jsky,$doti); + + # in => ucs4 + ok(esc($str->ucs4()),$ucs4,"$code=>ucs4"); + + # ucs4 => others + ok(esc($str->sjis()), $sjis, "$code=>ucs4=>sjis" ); + ok(esc($str->sjis_imode()),$imode,"$code=>ucs4=>imode"); + ok(esc($str->sjis_jsky()), $jsky, "$code=>ucs4=>jsky" ); + ok(esc($str->sjis_doti()), $doti, "$code=>ucs4=>doti" ); +} + +sub escl +{ + map{esc($_)}@_; +} +sub esc +{ + my $str = shift; + $str =~ s/\\/\\\\/g; + $str =~ s/\n/\\n/g; + $str =~ s/\e/\\e/g; + $str =~ s/\r/\\r/g; + $str =~ s/([\x00-\x1f])/'\x'.unpack("H*",$1)/ge; + $str; +} diff --git a/cpan/dist/Unicode-Japanese/t/fromutf8.t b/cpan/dist/Unicode-Japanese/t/fromutf8.t new file mode 100644 index 00000000..4262ac68 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/fromutf8.t @@ -0,0 +1,40 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 7 } + +## check from utf8 convert + +my $string; + +# sjis +$string = new Unicode::Japanese "\xe6\x84\x9b"; +ok($string->sjis, "\x88\xa4"); + +# euc +$string = new Unicode::Japanese "\xe6\x84\x9b"; +ok($string->euc, "\xb0\xa6"); + +# jis(iso-2022-jp) +$string = new Unicode::Japanese "\xe6\x84\x9b"; +ok($string->jis, "\x1b\x24\x42\x30\x26\x1b\x28\x42"); + +# imode +$string = new Unicode::Japanese "\xf3\xbf\xa2\xa8"; +ok($string->sjis_imode, "\xf8\xa8"); + +# dot-i +$string = new Unicode::Japanese "\xf3\xbf\x81\x88\xf3\xbf\x8e\x8e"; +ok($string->sjis_doti, "\xf0\x48\xf3\x8e"); + +# j-sky +$string = new Unicode::Japanese "\xf3\xbf\xb0\xb2"; +ok($string->sjis_jsky, "\e\$F2\x0f"); + +# j-sky(packed) +$string = new Unicode::Japanese "\xf3\xbf\xb0\xb2\xf3\xbf\xb1\x84"; +ok($string->sjis_jsky, "\e\$F2D\x0f"); + + diff --git a/cpan/dist/Unicode-Japanese/t/getcode.t b/cpan/dist/Unicode-Japanese/t/getcode.t new file mode 100644 index 00000000..e6d358ee --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/getcode.t @@ -0,0 +1,64 @@ + +use Test; + +use Unicode::Japanese qw(PurePerl); + +BEGIN { plan tests => 15 } + +## getcode method + +sub test +{ + my $src = shift; + my $icode = shift; + my $code = Unicode::Japanese->new->getcode($src); + ok($code, $icode, 'src:'.unpack('H*',$src)); +} + +my $code; + +$code = Unicode::Japanese->new->getcode("\x00\x00\xfe\xff"); +ok($code, 'utf32'); + +$code = Unicode::Japanese->new->getcode("\xff\xfe\x00\x00"); +ok($code, 'utf32'); + +$code = Unicode::Japanese->new->getcode("\xfe\xff"); +ok($code, 'utf16'); + +$code = Unicode::Japanese->new->getcode("\xff\xfe"); +ok($code, 'utf16'); + +$code = Unicode::Japanese->new->getcode("\x00\x00\x61\x1b"); +ok($code, 'utf32-be'); + +$code = Unicode::Japanese->new->getcode("\x1b\x61\x00\x00"); +ok($code, 'utf32-le'); + +$code = Unicode::Japanese->new->getcode("love"); +ok($code, 'ascii'); + +$code = Unicode::Japanese->new->getcode("\x1b\x24\x42\x30\x26\x1b\x28\x42"); +ok($code, 'jis'); + +$code = Unicode::Japanese->new->getcode("\e\$EE\x0f"); +ok($code, 'sjis-jsky'); + +$code = Unicode::Japanese->new->getcode("\xb0\xa6"); +ok($code, 'euc'); + +$code = Unicode::Japanese->new->getcode("\x88\xa4"); +ok($code, 'sjis'); + +test("\x88\xa4\xf8\xdf", 'sjis-imode'); + +$code = Unicode::Japanese->new->getcode("\x88\xa4\xf1\xb5"); +ok($code, 'sjis-doti'); + +$code = Unicode::Japanese->new->getcode("\xe6\x84\x9b"); +ok($code, 'utf8'); + +$code = Unicode::Japanese->new->getcode("\xcd\x10\x89\x01"); +ok($code, 'unknown'); + + diff --git a/cpan/dist/Unicode-Japanese/t/hirakata.t b/cpan/dist/Unicode-Japanese/t/hirakata.t new file mode 100644 index 00000000..68165973 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/hirakata.t @@ -0,0 +1,22 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 2 } + +## convert hiragana <-> katakana + +my $string; + +# hiragana -> katakana +$string = new Unicode::Japanese "\xe3\x81\x82\xe3\x81\x84\xe3\x81\x86"; +$string->hira2kata; +ok($string->get, "\xe3\x82\xa2\xe3\x82\xa4\xe3\x82\xa6"); + +# katakana -> hiragana +$string = new Unicode::Japanese "\xe3\x82\xa2\xe3\x82\xa4\xe3\x82\xa6"; +$string->kata2hira; +ok($string->get, "\xe3\x81\x82\xe3\x81\x84\xe3\x81\x86"); + + diff --git a/cpan/dist/Unicode-Japanese/t/illlet.t b/cpan/dist/Unicode-Japanese/t/illlet.t new file mode 100644 index 00000000..f1dfb92d --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/illlet.t @@ -0,0 +1,35 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 6 } + +## convert an illustrated letter between different types + +my $string; + +# dot-i/j-sky to imode +$string = new Unicode::Japanese "\xf3\xbf\x81\x88\xf3\xbf\x8e\x8e"; +ok($string->sjis_imode, "\xf9\x8e\x82\xd2"); + +$string = new Unicode::Japanese "\xf3\xbf\xb0\xb2\xf3\xbf\xb1\x84"; +ok($string->sjis_imode, "\xf9\x82\xf9\x8f"); + + +# imode/j-sky to dot-i +$string = new Unicode::Japanese "\xf3\xbf\xa2\xa8"; +ok($string->sjis_doti, "\xf0\x76"); + +$string = new Unicode::Japanese "\xf3\xbf\xb0\xb2\xf3\xbf\xb1\x84"; +ok($string->sjis_doti, "\xf4\xa8\xf0\x49"); + +# imode/dot-i to j-sky +$string = new Unicode::Japanese "\xf3\xbf\xa2\xa8"; +ok($string->sjis_jsky, "\x1b\x24\x46\x60\x0f"); + +# U+? U+000ff38e +$string = new Unicode::Japanese "\xf3\xbf\x81\x88\xf3\xbf\x8e\x8e"; +ok($string->sjis_jsky, "\x1b\x24\x46\x43\x0f\x82\xd2"); + + diff --git a/cpan/dist/Unicode-Japanese/t/outrange.t b/cpan/dist/Unicode-Japanese/t/outrange.t new file mode 100644 index 00000000..891e6d67 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/outrange.t @@ -0,0 +1,36 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 6 } + +## check from utf8 convert + +my $string; + +# sjis +$string = new Unicode::Japanese "\xe2\x99\xa5"; +ok($string->sjis, "♥", "U+2665 (9829) => sjis"); + +# euc +$string = new Unicode::Japanese "\xe2\x99\xa5"; +ok($string->euc, "♥", "U+2665 (9829) => eucjp"); + +# jis(iso-2022-jp) +$string = new Unicode::Japanese "\xe2\x99\xa5"; +ok($string->jis, "♥", "U+2665 (9829) => jis"); + +# imode +$string = new Unicode::Japanese "\xe2\x99\xa5"; +ok($string->sjis_imode, "?", "U+2665 (9829) => imode"); + +# dot-i +$string = new Unicode::Japanese "\xe2\x99\xa5"; +ok($string->sjis_doti, "?", "U+2665 (9829) => doti"); + +# j-sky +$string = new Unicode::Japanese "\xe2\x99\xa5"; +ok($string->sjis_jsky, "?", "U+2665 (9829) => jsky"); + + diff --git a/cpan/dist/Unicode-Japanese/t/random.pl b/cpan/dist/Unicode-Japanese/t/random.pl new file mode 100644 index 00000000..2cac166e --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/random.pl @@ -0,0 +1,72 @@ +#!/usr/bin/perl +# +# t/random.pl +# +# ランダムに作成した文字列(バイナリ列)をいろいろ変換. +# +# $ sh runtest.sh t/random.pl > random.out +# +# 異常終了時には +# $ sh runtest.sh t/random_redo.pl +# でリトライできます. +# +use strict; +use Unicode::Japanese; + +my $maxlen = 512; + +our @charcodes = ( + 'jis', 'sjis', 'euc', + 'sjis-imode', 'sjis-doti', 'sjis-jsky', + ); + +our $count = 0; + +my $file = 'random.dat'; +open(FILE,">$file") or die "cannot open [$file]"; +select((select(FILE),$|=1)[0]); + +$| = 1; +print "[0x000000]"; + +for(;; ++$count) +{ + if( ($count&0xFF)==0 && $count ) + { + if( ($count&0x3FFF)==0 ) + { + printf "\n[%#08x]",$count; + }else + { + print "."; + } + } + + my $len = int(rand($maxlen-4))+4; + my $src = ''; + for( my $i=0; $i<$len; ++$i ) + { + $src .= pack('C',int(rand(0x256))); + } + + seek FILE,0,0; + print FILE pack('N',$count); + print FILE pack('N',length($src)); + print FILE $src; + truncate FILE,length($src)+8; + + # ------------------------------------ + # utf8 => jis/eucjp/etc. + # + my $str = Unicode::Japanese->new($src,'utf8'); + foreach my $ocode ( @charcodes ) + { + $str->conv($ocode); + } + # ------------------------------------ + # jis/eucjp/etc. => utf8 + foreach my $icode ( @charcodes ) + { + Unicode::Japanese->new($src,$icode); + } +} diff --git a/cpan/dist/Unicode-Japanese/t/random_redo.pl b/cpan/dist/Unicode-Japanese/t/random_redo.pl new file mode 100644 index 00000000..21a9bf6b --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/random_redo.pl @@ -0,0 +1,41 @@ +#!/usr/bin/perl + +use strict; +use Unicode::Japanese; + +our @charcodes = ( + 'jis', 'sjis', 'euc', + 'sjis-imode', 'sjis-doti', 'sjis-jsky', + ); + +my $file = 'random.dat'; +open(FILE,"<$file") or die "cannot open [$file]"; + +my $dat; +read FILE,$dat,8; +my ($count,$len) = unpack('NN',$dat); +printf "[%#08x] len:%d\n",$count,$len; +read FILE,$dat,$len; + +{ + my $src = $dat; + + # ------------------------------------ + # utf8 => jis/eucjp/etc. + # + my $str = Unicode::Japanese->new($src,'utf8'); + foreach my $ocode ( @charcodes ) + { + print "utf8=>$ocode...\n"; + $str->conv($ocode); + } + # ------------------------------------ + # jis/eucjp/etc. => utf8 + foreach my $icode ( @charcodes ) + { + print "$icode=>utf8...\n"; + Unicode::Japanese->new($src,$icode); + } +} + +print "done\n"; diff --git a/cpan/dist/Unicode-Japanese/t/toutf8.t b/cpan/dist/Unicode-Japanese/t/toutf8.t new file mode 100644 index 00000000..e8c24e40 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/toutf8.t @@ -0,0 +1,39 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 7 } + +## check to utf8 convert + +my $string; + +# sjis +$string = new Unicode::Japanese "\x88\xa4", 'sjis'; +ok($string->get, "\xe6\x84\x9b"); + +# euc +$string = new Unicode::Japanese "\xb0\xa6", 'euc'; +ok($string->get, "\xe6\x84\x9b"); + +# jis(iso-2022-jp) +$string = new Unicode::Japanese "\x1b\x24\x42\x30\x26\x1b\x28\x42", 'jis'; +ok($string->get, "\xe6\x84\x9b"); + +# imode +$string = new Unicode::Japanese "\xf8\xa8", 'sjis-imode'; +ok($string->get, "\xf3\xbf\xa2\xa8"); + +# dot-i +$string = new Unicode::Japanese "\xf0\x48\xf3\x8e", 'sjis-doti'; +ok($string->get, "\xf3\xbf\x81\x88\xf3\xbf\x8e\x8e"); + +# j-sky +$string = new Unicode::Japanese "\e\$F2\x0f", 'sjis-jsky'; +ok($string->get, "\xf3\xbf\xb0\xb2"); + +# j-sky(packed) +$string = new Unicode::Japanese "\e\$F2D\x0f", 'sjis-jsky'; +ok($string->get, "\xf3\xbf\xb0\xb2\xf3\xbf\xb1\x84"); + diff --git a/cpan/dist/Unicode-Japanese/t/ucs2_utf8.pl b/cpan/dist/Unicode-Japanese/t/ucs2_utf8.pl new file mode 100644 index 00000000..d60419ff --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/ucs2_utf8.pl @@ -0,0 +1,129 @@ +#!/usr/bin/perl +# +# ucs2 <=> utf8 全文字チェック +# ucs2(0x0000..0xFFFF) => utf8 +# utf8(0x000000..0xFFFFFF) => ucs2 +# + +use strict; +use Unicode::Japanese; + +$| = 1; + +# ucs2 => utf8 +print "ucs2 => utf8\n"; +print "[0x000000]"; +for( my $i=0; $i<=0xFFFF; ++$i ) +{ + if( ($i&0xFF)==0 && $i ) + { + if( ($i&0x3FFF)==0 ) + { + printf "\n[%#08x]",$i; + }else + { + print "."; + } + } + + my $src = pack('n',$i); + + my $str = Unicode::Japanese->new($src,'ucs2'); + my $xs = $str->utf8(); + my $orig = _ucs2_utf8($str,$src); + if( $xs ne $orig ) + { + $src = unpack('H*',$src); + $xs = unpack('H*',$xs); + $orig = unpack('H*',$orig); + print "\n"; + die "not match, src:[$src], xs:[$xs] != orig:[$orig]"; + } +} +print "\n"; + +# utf8 => ucs2 +print "utf8 => ucs2\n"; +print "[0x000000]"; +for( my $i=0; $i<=0xFFFFFF; ++$i ) +{ + if( ($i&0xFF)==0 && $i ) + { + if( ($i&0x3FFF)==0 ) + { + printf "\n[%#08x]",$i; + }else + { + print "."; + } + } + + my $src = pack('N',$i); + $src =~ s/^\0+//; + + my $str = Unicode::Japanese->new($src,'utf8'); + my $xs = $str->ucs2(); + my $orig = _utf8_ucs2($str,$src); + if( $xs ne $orig ) + { + $src = unpack('H*',$src); + $xs = unpack('H*',$xs); + $orig = unpack('H*',$orig); + print "\n"; + die "not match, src:[$src], xs:[$xs] != orig:[$orig]"; + } +} +print "\n"; + + +# ---------------------------------------------------------------------- + +my @U2T; +my %T2U; + +sub _ucs2_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + for my $uc (unpack("n*", $str)) + { + $result .= $U2T[$uc] ? $U2T[$uc] : + ($U2T[$uc] = ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . + chr(0x80 | ($uc & 0x3F))); + } + + $result; +} + +sub _utf8_ucs2 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}|(.))/ + defined($2)?"\0$2": + $T2U{$1} + or ($T2U{$1} + = ((length($1) == 1) ? pack("n", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("n", (($c1 & 0x1F)<<6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("n", (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F))) : "\0?")) + /eg; + $str; +} diff --git a/cpan/dist/Unicode-Japanese/t/util.t b/cpan/dist/Unicode-Japanese/t/util.t new file mode 100644 index 00000000..3951be5e --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/util.t @@ -0,0 +1,29 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 4 } + +## Util method + +my $string; + +# strlen +$string = new Unicode::Japanese "\xe3\x82\xa2\xe3\x82\xa4\xe3\x82\xa6"; +ok($string->strlen, 6); + +# strcut +$string = new Unicode::Japanese "\xe3\x82\xa2\xe3\x82\xa4\xe3\x82\xa6"; +ok($string->strcut(5)->[0], "\xe3\x82\xa2\xe3\x82\xa4"); + +# join_csv +$string = new Unicode::Japanese; +$string->join_csv([1, 2, 'abc', '"123"']); +ok($string->get, '1,2,abc,"""123"""' . "\n"); + +# split_csv +$string = new Unicode::Japanese '1,2,abc,"""123"""'; +ok($string->split_csv->[3], '"123"'); + + diff --git a/cpan/dist/Unicode-Japanese/t/verify_sjis_ucs2.pl b/cpan/dist/Unicode-Japanese/t/verify_sjis_ucs2.pl new file mode 100644 index 00000000..c935103e --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/verify_sjis_ucs2.pl @@ -0,0 +1,198 @@ +#!/usr/bin/perl -w +# +# t/verify_sjis_ucs2.pl +# +# sjis=>ucs2とucs2=>sjisの全文字テスト +# XS側だけのてすと…. +# +# $ sh runtest.sh t/verify_sjis_ucs2.pl +# +# all sjis(0x0000-0xFFFF) => ucs2 +# all ucs2(0x0000-0xFFFF) => sjis +# + +use strict; +#BEGIN{$Unicode::Japanese::PurePerl = 1;} +use Unicode::Japanese; +use IO::File; + +print "loading Uni::Jp\n"; +Unicode::Japanese->new(''); +my $msg = $Unicode::Japanese::xs_loaderror; +print "xs-load-message : [".(defined($msg)?$msg:'')."]".(!defined($msg)?' (undef)':$msg eq ''?' (empty)':'')."\n"; + +my $tablefh = new IO::File 'jcode/CP932.TXT' + or die "cannot open 'jcode/CP932.TXT'"; +print "reading 'jcode/CP932.TXT'...\n"; + +my(%s2u,%u2s); + +while(<$tablefh>) + { + next if(m/^#/); + next if(m/^$/); + + chomp; + + m/^0x([0-9a-fA-F]+)\s+(?:0x([0-9a-fA-F]+))?/ or die $_; + next if(!defined($2)); + + $s2u{hex($1)} = hex($2); + # CP932 Unicode + } + +%u2s = reverse(%s2u); + +$| = 1; + +# -------------------------------------------------------------------- +# 不一致時に出力する用 +sub dumpstr($$) +{ + my($hdr,$str)=@_; + my $line = $hdr.sprintf(" : [len:%d]",length($str)); + for( my $i=0; $i<length($str); ++$i ) + { + $line .= sprintf(" %02x",unpack('C',substr($str,$i,1))); + } + print STDERR $line." : $str\r\n"; +} + +# -------------------------------------------------------------------- +# tests sjis to ucs2 + +print "Testing sjis=>ucs2...\n"; +test_sjis_ucs2(); + +sub upack +{ + pack('n',shift); +} +sub sjis_ucs2 +{ + my $code = shift; + my $str = $code<=0xFF?pack("C",$code):pack('n',$code); + exists($s2u{$code}) ? upack($s2u{$code}) : + $code<=0xFF ? "\0?" : + $str =~ /^[\x81-\x9F\xE0-\xFC][\x40-\x7E\x80-\xFC]$/ ? "\0?" : + ( + (exists($s2u{$code>>8}) ? upack($s2u{$code>>8}) : "\0?"). + (exists($s2u{$code&255}) ? upack($s2u{$code&255}) : "\0?") + ) +} +sub test_sjis_ucs2 +{ + printf "[0x%#04x]",0; + for( my $i=0x0; $i<=0xffff; ++$i ) + { + if( ($i&0xFF)==0 && $i) + { + if( ($i&0x3FFF)==0 ) + { + printf "\n[%#06x]",$i; + }else + { + print '.'; + } + } + my $src = pack($i<=0xff?'c':'n',$i); + my $code = $i; + my $xs = Unicode::Japanese->new($src,'sjis')->ucs2(); + my $test = sjis_ucs2($code); + if( $xs ne $test ) + { + print STDERR "\n"; + print STDERR "<<sjis=>utf8>>\n"; + print STDERR "i : $i\n"; + dumpstr('sjis',$src); + dumpstr('xs ',$xs); + dumpstr('test',$test); + exit; + } + } + print "\n"; +} + +# -------------------------------------------------------------------- +# tests ucs2 to sjis + +print "Testing ucs2=>sjis...\n"; +test_ucs2_sjis(); + +sub spack +{ + my $code = shift; + $code <= 0xFF ? pack('C',$code) : pack('n',$code); +} +sub ucs2_sjis +{ + my $code = shift; + exists($u2s{$code}) ? spack($u2s{$code}) : + $code<=0x7F ? chr($code) : + '&#'.$code.';'; +} + +sub test_ucs2_sjis +{ + printf "[0x%#04x]",0; + for( my $i=0x0; $i<=0xffff; ++$i ) + { + if( ($i&0xFF)==0 && $i) + { + if( ($i&0x3FFF)==0 ) + { + printf "\n[%#06x]",$i; + }else + { + print '.'; + } + } + my $code = $i; + my $ucs2 = pack('n',$code); + my $xs = Unicode::Japanese->new($ucs2,'ucs2')->sjis(); + my $test = ucs2_sjis($code); + if( $xs ne $test ) + { + print STDERR "\n"; + print STDERR "<<utf8=>sjis>>\n"; + printf STDERR "i : 0x%04x\n",$i; + dumpstr('ucs2',$ucs2); + dumpstr('xs ',$xs); + dumpstr('test',$test); + exit; + } + } + print "\n"; +} + +# -------------------------------------------------------------------- +# done + +print "done\n"; + +# -------------------------------------------------------------------- +# PurePerl code, copy from String.pl +# +use vars qw(@U2T); + +sub _ucs2_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + for my $uc (unpack("n*", $str)) + { + $result .= $U2T[$uc] ? $U2T[$uc] : + ($U2T[$uc] = ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . + chr(0x80 | ($uc & 0x3F))); + } + + $result; +} diff --git a/cpan/dist/Unicode-Japanese/t/z2h.t b/cpan/dist/Unicode-Japanese/t/z2h.t new file mode 100644 index 00000000..2144039b --- /dev/null +++ b/cpan/dist/Unicode-Japanese/t/z2h.t @@ -0,0 +1,52 @@ + +use Test; + +use Unicode::Japanese; + +BEGIN { plan tests => 8 } + + +## h2z/z2h convert + +# h2z num +$string = new Unicode::Japanese "0129"; +$string->h2z; +ok($string->get, "\xef\xbc\x90\xef\xbc\x91\xef\xbc\x92\xef\xbc\x99"); + +# h2z alpha +$string = new Unicode::Japanese "abzABZ"; +$string->h2z; +ok($string->get, "\xef\xbd\x81\xef\xbd\x82\xef\xbd\x9a\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xba"); + +# h2z symbol +$string = new Unicode::Japanese "!#^*(-+~{]>?"; +$string->h2z; +ok($string->get, "\xef\xbc\x81\xef\xbc\x83\xef\xbc\xbe\xef\xbc\x8a\xef\xbc\x88\xe2\x88\x92\xef\xbc\x8b\xe3\x80\x9c\xef\xbd\x9b\xef\xbc\xbd\xef\xbc\x9e\xef\xbc\x9f"); + +# h2z kana +$string = new Unicode::Japanese "\xef\xbd\xa1\xef\xbd\xab\xe3\x81\x89\xef\xbd\xb3\xef\xbe\x9e"; +$string->h2z; +ok($string->get, "\xe3\x80\x82\xe3\x82\xa9\xe3\x81\x89\xe3\x83\xb4"); + +# z2h num +$string = new Unicode::Japanese "\xef\xbc\x90\xef\xbc\x91\xef\xbc\x92\xef\xbc\x99"; +$string->z2h; +ok($string->get, "0129"); + +# z2h alpha +$string = new Unicode::Japanese "\xef\xbd\x81\xef\xbd\x82\xef\xbd\x9a\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xba"; +$string->z2h; +ok($string->get, "abzABZ"); + +# z2h symbol +$string = new Unicode::Japanese "\xef\xbc\x81\xef\xbc\x83\xef\xbc\xbe\xef\xbc\x8a\xef\xbc\x88\xe2\x88\x92\xef\xbc\x8b\xe3\x80\x9c\xef\xbd\x9b\xef\xbc\xbd\xef\xbc\x9e\xef\xbc\x9f"; +$string->z2h; +ok($string->get, "!#^*(-+~{]>?"); + +# z2h kana +$string = new Unicode::Japanese "\xe3\x80\x82\xe3\x82\xa9\xe3\x81\x89\xe3\x83\xb4"; +$string->z2h; +ok($string->get, "\xef\xbd\xa1\xef\xbd\xab\xe3\x81\x89\xef\xbd\xb3\xef\xbe\x9e"); + + + diff --git a/cpan/dist/Unicode-Japanese/ucs2_utf8.cpp b/cpan/dist/Unicode-Japanese/ucs2_utf8.cpp new file mode 100644 index 00000000..fe6ac181 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/ucs2_utf8.cpp @@ -0,0 +1,171 @@ + +// $Id: ucs2_utf8.cpp,v 1.5 2002/07/01 00:09:54 hio Exp $ + +#include "Japanese.h" + +/* + * ucs2=>utf8文字列変換 + */ +EXTERN_C +SV* +xs_ucs2_utf8(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVpvn("",0); + } + STRLEN PL_na; + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + const int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)ucs2_utf8\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len*3/2+4); + + if( len&1 ) + { + Perl_croak(aTHX_ "Unicode::Japanese::ucs2_utf8, invalid length (not 2*n)"); + } + + const unsigned char* src_end = src+(len&~1); + + unsigned char buf[4]; + for(; src<src_end; src+=2 ) + { + const unsigned short ucs2 = ntohs(*(unsigned short*)src); + if( ucs2<0x80 ) + { + buf[0] = (unsigned char)ucs2; + result.append(buf,1); + }else if( ucs2<0x800 ) + { + buf[0] = 0xC0 | (ucs2 >> 6); + buf[1] = 0x80 | (ucs2 & 0x3F); + result.append(buf,2); + }else + { + buf[0] = 0xE0 | (ucs2 >> 12); + buf[1] = 0x80 | ((ucs2 >> 6) & 0x3F); + buf[2] = 0x80 | (ucs2 & 0x3F); + result.append(buf,3); + } + } + + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} + +/* + * utf8=>ucs2文字列変換 + */ +EXTERN_C +SV* +xs_utf8_ucs2(SV* sv_str) +{ + if( sv_str==&PL_sv_undef ) + { + return newSVpvn("",0); + } + STRLEN PL_na; + unsigned char* src = (unsigned char*)SvPV(sv_str,PL_na); + const int len = sv_len(sv_str); + + //fprintf(stderr,"Unicode::Japanese::(xs)utf8_ucs2\n",len); + //bin_dump("in ",src,len); + + SV_Buf result(len); + + const unsigned char* src_end = src+len; + + while( src<src_end ) + { + if( *src<=0x7f ) + { + result.append_ch2(htons(*src++)); + continue; + } + int utf8_len,ucs2; + if( 0xc0<=*src && *src<=0xdf ) + { // length [2] + utf8_len = 2; + if( src+1>=src_end || + src[1]<0x80 || 0xbf<src[1] ) + { + result.append_ch2(htons(*src++)); + continue; + } + ucs2 = ((src[0] & 0x1F)<<6)|(src[1] & 0x3F); + }else if( 0xe0<=*src && *src<=0xef ) + { // length [3] + utf8_len = 3; + if( src+2>=src_end || + src[1]<0x80 || 0xbf<src[1] || + src[2]<0x80 || 0xbf<src[2] ) + { + result.append_ch2(htons(*src++)); + continue; + } + ucs2 = ((src[0] & 0x0F)<<12)|((src[1] & 0x3F)<<6)|(src[2] & 0x3F); + }else if( 0xf0<=*src && *src<=0xf7 ) + { // length [4] + utf8_len = 4; + if( src+3>=src_end || + src[1]<0x80 || 0xbf<src[1] || + src[2]<0x80 || 0xbf<src[2] || + src[3]<0x80 || 0xbf<src[3] ) + { + result.append_ch2(htons(*src++)); + continue; + } + ucs2 = ((src[0] & 0x07)<<18)|((src[1] & 0x3F)<<12)| + ((src[2] & 0x3f) << 6)|(src[3] & 0x3F); + }else if( 0xf8<=*src && *src<=0xfb ) + { // length [5] + utf8_len = 5; + if( src+4>=src_end || + src[1]<0x80 || 0xbf<src[1] || + src[2]<0x80 || 0xbf<src[2] || + src[3]<0x80 || 0xbf<src[3] || + src[4]<0x80 || 0xbf<src[4] ) + { + result.append_ch2(htons(*src++)); + continue; + } + }else if( 0xfc<=*src && *src<=0xfd ) + { // length [6] + utf8_len = 6; + if( src+5>=src_end || + src[1]<0x80 || 0xbf<src[1] || + src[2]<0x80 || 0xbf<src[2] || + src[3]<0x80 || 0xbf<src[3] || + src[4]<0x80 || 0xbf<src[4] || + src[5]<0x80 || 0xbf<src[5] ) + { + result.append_ch2(htons(*src++)); + continue; + } + }else + { // invalid + result.append_ch2(htons(*src++)); + continue; + } + + if( ucs2 & ~0xFFFF ) + { // ucs2の範囲外 (ucs4の範囲) + result.append_ch2(htons('?')); + src += utf8_len; + continue; + } + result.append_ch2(htons(ucs2)); + src += utf8_len; + //bin_dump("now",dst_begin,dst-dst_begin); + } + + //bin_dump("out",result.getBegin(),result.getLength()); + result.setLength(); + + return result.getSv(); +} diff --git a/cpan/dist/Unicode-Japanese/xs_test.c b/cpan/dist/Unicode-Japanese/xs_test.c new file mode 100644 index 00000000..b29903e0 --- /dev/null +++ b/cpan/dist/Unicode-Japanese/xs_test.c @@ -0,0 +1,28 @@ + +/* $Id: xs_test.c,v 1.1 2002/01/18 17:17:19 hio Exp $ */ + +#include "mediate.h" +#include <unistd.h> // memmap +#include <sys/mman.h> // memmap +#include <sys/stat.h> // stat +#include <fcntl.h> // open + +#ifndef MAP_FAILED +#define MAP_FAILED ((void*)-1) +#endif + +void* do_memmap(char* filepath) +{ + int fd = open(filepath,O_RDONLY|O_NONBLOCK); + struct stat st; + int res = fstat(fd,&st); + void* ptr = mmap(NULL,st.st_size,PROT_READ,MAP_PRIVATE,fd,0); + close(fd); + return ptr; +} + +void do_unmemmap(void* ptr) +{ + munmap(ptr,0); +} + diff --git a/cpan/lib/Unicode/Japanese.pm b/cpan/lib/Unicode/Japanese.pm new file mode 100644 index 00000000..a61fee02 --- /dev/null +++ b/cpan/lib/Unicode/Japanese.pm @@ -0,0 +1,2809 @@ +package Unicode::Japanese; +# $Id: Japanese_stub.pm,v 1.25 2002/06/30 23:20:17 hio Exp $ + +use strict; +use vars qw($VERSION $PurePerl $xs_loaderror); +$VERSION = '0.09'; + +sub import +{ + my $pkg = shift; + my @na = grep{ !/^PurePerl$/i }@_; + if( @na != @_ ) + { + $PurePerl = 1; + } + if( @na ) + { + use Carp; + croak("invalid parameter (".join(',',@na).")"); + } +} + +sub DESTROY +{ +} + +sub load_xs +{ + #print STDERR "load_xs\n"; + if( $PurePerl ) + { + #print STDERR "PurePerl mode\n"; + $xs_loaderror = 'disabled'; + return; + } + #print STDERR "XS mode\n"; + + my $use_xs; + LoadXS: + { + + #print STDERR "* * bootstrap...\n"; + eval q + { + use strict; + require DynaLoader; + use vars qw(@ISA); + @ISA = qw(DynaLoader); + local($SIG{__DIE__}) = 'DEFAULT'; + Unicode::Japanese->bootstrap($VERSION); + }; + #print STDERR "* * try done.\n"; + #undef @ISA; + if( $@ ) + { + #print STDERR "failed.\n"; + #print STDERR "$@\n"; + $use_xs = 0; + $xs_loaderror = $@; + undef $@; + last LoadXS; + } + #print STDERR "succeeded.\n"; + $use_xs = 1; + eval q + { + #print STDERR "over riding _s2u,_u2s\n"; + do_memmap(); + #print STDERR "memmap done\n"; + END{ do_memunmap(); } + #print STDERR "binding xsubs done.\n"; + }; + if( $@ ) + { + #print STDERR "error on last part of load XS.\n"; + $xs_loaderror = $@; + CORE::die($@); + } + + #print STDERR "done.\n"; + } + + if( $@ ) + { + $xs_loaderror = $@; + CORE::die("Cannot Load Unicode::Japanese either XS nor PurePerl\n$@"); + } + if( !$use_xs ) + { + #print STDERR "no xs.\n"; + eval q + { + sub do_memmap($){} + sub do_memunmap($){} + }; + } + $xs_loaderror = '' if( !defined($xs_loaderror) ); + #print STDERR "load_xs done.\n"; +} + +use vars qw($FH $TABLE $HEADLEN $PROGLEN); + +sub gensym { + package Unicode::Japanese::Symbol; + no strict; + $genpkg = "Unicode::Japanese::Symbol::"; + $genseq = 0; + my $name = "GEN" . $genseq++; + my $ref = \*{$genpkg . $name}; + delete $$genpkg{$name}; + $ref; +} + +sub _init_table { + + if(!defined($HEADLEN)) + { + $FH = gensym; + + my $file = "Unicode/Japanese.pm"; + OPEN: + { + foreach my $path (@INC) + { + my $mypath = $path; + $mypath =~ s#/$##; + if (-f "$mypath/$file") + { + open($FH,"$mypath/$file") || CORE::die; + binmode($FH); + last OPEN; + } + } + CORE::die "Can't find Japanese.pm in \@INC\n"; + } + + local($/) = "\n"; + my $line; + while($line = <$FH>) + { + last if($line =~ m/^__DATA__/); + } + $PROGLEN = tell($FH); + + read($FH, $HEADLEN, 4) + or die "Can't read table. [$!]\n"; + $HEADLEN = unpack('N', $HEADLEN); + read($FH, $TABLE, $HEADLEN) + or die "Can't seek table. [$!]\n"; + $TABLE = eval $TABLE; + if($@) + { + die "Internal Error. [$@]\n"; + } + if(!defined($TABLE)) + { + die "Internal Error.\n"; + } + $HEADLEN += 4; + + # load xs. + load_xs(); + } +} + +sub _getFile { + my $this = shift; + + my $file = shift; + +# print STDERR "_getFile($file, $TABLE->{$file}{offset}, $TABLE->{$file}{length})\n"; + seek($FH, $PROGLEN + $HEADLEN + $TABLE->{$file}{offset}, 0) + or die "Can't seek $file. [$!]\n"; + + my $data; + read($FH, $data, $TABLE->{$file}{length}) + or die "Can't read $file. [$!]\n"; + + $data; +} + +sub new +{ + my $pkg = shift; + my $this = {}; + + if( defined($pkg) ) + { + bless $this, $pkg; + $this->_init_table; + }else + { + bless $this; + } + + if(defined($_[0])) + { + $this->set(@_); + } + + $this; +} + + + +use vars qw(%CHARCODE %ESC %RE); +use vars qw(@J2S @S2J @S2E @E2S @U2T %T2U %S2U %U2S); + +%CHARCODE = ( + UNDEF_EUC => "\xa2\xae", + UNDEF_SJIS => "\x81\xac", + UNDEF_JIS => "\xa2\xf7", + UNDEF_UNICODE => "\x20\x20", + ); + +%ESC = ( + JIS_0208 => "\e\$B", + JIS_0212 => "\e\$(D", + ASC => "\e\(B", + KANA => "\e\(I", + E_JSKY_START => "\e\$", + E_JSKY_END => "\x0f", + ); + +%RE = + ( + ASCII => '[\x00-\x7f]', + EUC_0212 => '\x8f[\xa1-\xfe][\xa1-\xfe]', + EUC_C => '[\xa1-\xfe][\xa1-\xfe]', + EUC_KANA => '\x8e[\xa1-\xdf]', + JIS_0208 => '\e\$\@|\e\$B|\e&\@\e\$B', + JIS_0212 => "\e" . '\$\(D', + JIS_ASC => "\e" . '\([BJ]', + JIS_KANA => "\e" . '\(I', + SJIS_DBCS => '[\x81-\x9f\xe0-\xef\xfa-\xfc][\x40-\x7e\x80-\xfc]', + SJIS_KANA => '[\xa1-\xdf]', + UTF8 => '[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}', + BOM2_BE => '\xfe\xff', + BOM2_LE => '\xff\xfe', + BOM4_BE => '\x00\x00\xfe\xff', + BOM4_LE => '\xff\xfe\x00\x00', + UTF32_BE => '\x00[\x00-\x10][\x00-\xff]{2}', + UTF32_LE => '[\x00-\xff]{2}[\x00-\x10]\x00', + E_IMODE => '\xf8[\x9f-\xfc]|\xf9[\x40-\x49\x72-\x7e\x80-\xb0]', + E_JSKY1 => '[EFG]', + E_JSKY2 => '[\!-z]', + E_DOTI => '\xf0[\x40-\x7e\x80-\xfc]|\xf1[\x40-\x7e\x80-\xd6]|\xf2[\x40-\x7e\x80-\xab\xb0-\xd5\xdf-\xfc]|\xf3[\x40-\x7e\x80-\xfa]|\xf4[\x40-\x4f\x80\x84-\x8a\x8c-\x8e\x90\x94-\x96\x98-\x9c\xa0-\xa4\xa8-\xaf\xb4\xb5\xbc-\xbe\xc4\xc5\xc8\xcc]', + E_JSKY_START => quotemeta($ESC{E_JSKY_START}), + E_JSKY_END => quotemeta($ESC{E_JSKY_END}), + ); + +$RE{E_JSKY} = $RE{E_JSKY_START} + . $RE{E_JSKY1} . $RE{E_JSKY2} . '+' + . $RE{E_JSKY_END}; + +use vars qw($s2u_table $u2s_table); +use vars qw($ei2u $ed2u $ej2u $eu2i $eu2d $eu2j); + +# encode/decode + +use vars qw(%_h2zNum %_z2hNum %_h2zAlpha %_z2hAlpha %_h2zSym %_z2hSym %_h2zKanaK %_z2hKanaK %_h2zKanaD %_z2hKanaD %_hira2kata %_kata2hira); + + + +AUTOLOAD +{ + use strict; + use vars qw($AUTOLOAD); + + #print STDERR "AUTOLOAD... $AUTOLOAD\n"; + + my $save = $@; + my @BAK = @_; + + my $subname = $AUTOLOAD; + $subname =~ s/^Unicode\:\:Japanese\:\://; + + #print "subs..\n",join("\n",keys %$TABLE,''); + + # check + if(!defined($TABLE->{$subname}{offset})) + { + if (substr($AUTOLOAD,-9) eq '::DESTROY') + { + { + no strict; + *$AUTOLOAD = sub {}; + } + $@ = $save; + @_ = @BAK; + goto &$AUTOLOAD; + } + + CORE::die "Undefined subroutine \&$AUTOLOAD called.\n"; + } + if($TABLE->{$subname}{offset} == -1) + { + CORE::die "Double loaded \&$AUTOLOAD. It has some error.\n"; + } + + seek($FH, $PROGLEN + $HEADLEN + $TABLE->{$subname}{offset}, 0) + or die "Can't seek $subname. [$!]\n"; + + my $sub; + read($FH, $sub, $TABLE->{$subname}{length}) + or die "Can't read $subname. [$!]\n"; + + CORE::eval($sub); + if ($@) + { + CORE::die $@; + } + $DB::sub = $AUTOLOAD; # Now debugger know where we are. + + # evaled + $TABLE->{$subname}{offset} = -1; + + $@ = $save; + @_ = @BAK; + goto &$AUTOLOAD; +} + + +1; + +=head1 NAME + +Unicode::Japanese - Japanese Character Encoding Handler + +=head1 SYNOPSIS + +use Unicode::Japanese; + +# convert utf8 -> sjis + +print Unicode::Japanese->new($str)->sjis; + +# convert sjis -> utf8 + +print Unicode::Japanese->new($str,'sjis')->get; + +# convert sjis (imode_EMOJI) -> utf8 + +print Unicode::Japanese->new($str,'sjis-imode')->get; + +# convert ZENKAKU (utf8) -> HANKAKU (utf8) + +print Unicode::Japanese->new($str)->z2h->get; + +=head1 DESCRIPTION + +Module for conversion among Japanese character encodings. + +=head2 FEATURES + +=over 2 + +=item * + +The instance stores internal strings in UTF-8. + +=item * + +Supports both XS and Non-XS. +Use XS for high performance, +or No-XS for ease to use (only by copying Japanese.pm). + +=item * + +Supports conversion between ZENKAKU and HANKAKU. + +=item * + +Safely handles "EMOJI" of the mobile phones (DoCoMo i-mode, ASTEL dot-i +and J-PHONE J-Sky) by mapping them on Unicode Private Use Area. + +=item * + +Supports conversion of the same image of EMOJI +between different mobile phone's standard mutually. + +=item * + +Considers Shift_JIS(SJIS) as MS-CP932. +(Shift_JIS on MS-Windows (MS-SJIS/MS-CP932) differ from +generic Shift_JIS encodings.) + +=item * + +On converting Unicode to SJIS (and EUC-JP/JIS), those encodings that cannot +be converted to SJIS (except "EMOJI") are escaped in "&#dddd;" format. +"EMOJI" on Unicode Private Use Area is going to be '?'. +When converting strings from Unicode to SJIS of mobile phones, +any characters not up to their standard is going to be '?' + +=back + +=head1 METHODS + +=over 4 + +=item $s = Unicode::Japanese->new($str [, $icode [, $encode]]) + +Creates a new instance of Unicode::Japanese. + +If arguments are specified, passes through to set method. + +=item $s->set($str [, $icode [, $encode]]) + +=over 2 + +=item $str: string + +=item $icode: character encodings, may be omitted (default = 'utf8') + +=item $encode: ASCII encoding, may be omitted. + +=back + +Set a string in the instance. +If '$icode' is omitted, string is considered as UTF-8. + +To specify a encodings, choose from the following; +'jis', 'sjis', 'euc', 'utf8', +'ucs2', 'ucs4', 'utf16', 'utf16-ge', 'utf16-le', +'utf32', 'utf32-ge', 'utf32-le', 'ascii', 'binary', +'sjis-imode', 'sjis-doti', 'sjis-jsky'. + +'&#dddd' will be converted to "EMOJI", when specified 'sjis-imode' +or 'sjis-doti'. + +For auto encoding detection, you MUST specify 'auto' +so as to call getcode() method automatically. + +For ASCII encoding, only 'base64' may be specified. +With it, the string will be decoded before storing. + +To decode binary, specify 'binary' as the encoding. + +=item $str = $s->get + +=over 2 + +=item $str: string (UTF-8) + +=back + +Gets a string with UTF-8. + +=item $code = $s->getcode($str) + +=over 2 + +=item $str: string + +=item $code: character encoding name + +=back + +Detects the character encodings of I<$str>. + +Notice: This method detects B<NOT> encoding of the string in the instance +but I<$str>. + +Character encodings are distinguished by the following algorithm: + +(In case of PurePerl) + +=over 4 + +=item 1 + +If BOM of UTF-32 is found, the encoding is utf32. + +=item 2 + +If BOM of UTF-16 is found, the encoding is utf16. + +=item 3 + +If it is in proper UTF-32BE, the encoding is utf32-be. + +=item 4 + +If it is in proper UTF-32LE, the encoding is utf32-le. + +=item 5 + +Without NON-ASCII characters, the encoding is ascii. +(control codes except escape sequences has been included in ASCII) + +=item 6 + +If it includes ISO-2022-JP(JIS) escape sequences, the encoding is jis. + +=item 7 + +If it includes "J-PHONE EMOJI", the encoding is sjis-sky. + +=item 8 + +If it is in proper EUC-JP, the encoding is euc. + +=item 9 + +If it is in proper SJIS, the encoding is sjis. + +=item 10 + +If it is in proper SJIS and "EMOJI" of i-mode, the encoding is sjis-imode. + +=item 11 + +If it is in proper SJIS and "EMOJI" of dot-i,the encoding is sjis-doti. + +=item 12 + +If it is in proper UTF-8, the encoding is utf8. + +=item 13 + +If none above is true, the encoding is unknown. + +=back + +(In case of XS) + +=over 4 + +=item 1 + +If BOM of UTF-32 is found, the encoding is utf32. + +=item 2 + +If BOM of UTF-16 is found, the encoding is utf16. + +=item 3 + +String is checked by State Transition if it is applicable +for any listed encodings below. + +ascii / euc-jp / sjis / jis / utf8 / utf32-be / utf32-le / sjis-jsky / +sjis-imode / sjis-doti + +=item 4 + +The listed order below is applied for a final determination. + +utf32-be / utf32-le / ascii / jis / euc-jp / sjis / sjis-jsky / sjis-imode / +sjis-doti / utf8 + +=item 5 + +If none above is true, the encoding is unknown. + + +=back + +Regarding the algorithm, pay attention to the following: + +=over 2 + +=item * + +UTF-8 is occasionally detected as SJIS. + +=item * + +Can NOT detect UCS2 automatically. + +=item * + +Can detect UTF-16 only when the string has BOM. + +=item * + +Can detect "EMOJI" when it is stored in binary, not in "&#dddd;" +format. (If only stored in "&#dddd;" format, getcode() will +return incorrect result. In that case, "EMOJI" will be crashed.) + +=back + +Because each of XS and PurePerl has a different algorithm, A result of +the detection would be possibly different. In case that the string is +SJIS with escape characters, it would be considered as SJIS on +PurePerl. However, it can't be detected as S-JIS on XS. This is +because by using Algorithm, the string can't be distinguished between +SJIS and SJIS-Jsky. This exclusion of escape characters on XS from +the detection is suppose to be the same for EUC-JP. + +=item $str = $s->conv($ocode, $encode) + +=over 2 + +=item $ocode: output character encoding (Choose from 'jis', 'sjis', 'euc', 'utf8', 'ucs2', 'ucs4', 'utf16', 'binary') + +=item $encode: ASCII encoding, may be omitted. + +=item $str: string + +=back + +Gets a string converted to I<$ocode>. + +For ASCII encoding, only 'base64' may be specified. With it, the string +encoded in base64 will be returned. + +=item $s->tag2bin + +Replaces the substrings "&#dddd;" in the string with the binary entity +they mean. + +=item $s->z2h + +Converts ZENKAKU to HANKAKU. + +=item $s->h2z + +Converts HANKAKU to ZENKAKU. + +=item $s->hira2kata + +Converts HIRAGANA to KATAKANA. + +=item $s->kata2hira + +Converts KATAKANA to HIRAGANA. + +=item $str = $s->jis + +$str: string (JIS) + +Gets the string converted to ISO-2022-JP(JIS). + +=item $str = $s->euc + +$str: string (EUC-JP) + +Gets the string converted to EUC-JP. + +=item $str = $s->utf8 + +$str: string (UTF-8) + +Gets the string converted to UTF-8. + +=item $str = $s->ucs2 + +$str: string (UCS2) + +Gets the string converted to UCS2. + +=item $str = $s->ucs4 + +$str: string (UCS4) + +Gets the string converted to UCS4. + +=item $str = $s->utf16 + +$str: string (UTF-16) + +Gets the string converted to UTF-16(big-endian). +BOM is not added. + +=item $str = $s->sjis + +$str: string (SJIS) + +Gets the string converted to Shift_JIS(MS-SJIS/MS-CP932). + +=item $str = $s->sjis_imode + +$str: string (SJIS/imode_EMOJI) + +Gets the string converted to SJIS for i-mode. + +=item $str = $s->sjis_doti + +$str: string (SJIS/dot-i_EMOJI) + +Gets the string converted to SJIS for dot-i. + +=item $str = $s->sjis_sky + +$str: string (SJIS/J-SKY_EMOJI) + +Gets the string converted to SJIS for j-sky. + +=item @str = $s->strcut($len) + +=over 2 + +=item $len: number of characters + +=item @str: strings + +=back + +Splits the string by length(I<$len>). + +=item $len = $s->strlen + +$len: `visual width' of the string + +Gets the length of the string. This method has been offered to +substitute for perl build-in length(). ZENKAKU characters are +assumed to have lengths of 2, regardless of the coding being +SJIS or UTF-8. + +=item $s->join_csv(@values); + +@values: data array + +Converts the array to a string in CSV format, then stores into the instance. +In the meantime, adds a newline("\n") at the end of string. + +=item @values = $s->split_csv; + +@values: data array + +Splits the string, accounting it is in CSV format. +Each newline("\n") is removed before split. + +=back + + +=head1 DESCRIPTION OF UNICODE MAPPING + +=over 2 + +=item SJIS + +Mapped as MS-CP932. Mapping table in the following URL is used. + +ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT + +If a character cannot be mapped to SJIS from Unicode, +it will be converted to &#dddd; format. + +Also, any unmapped character will be converted into "?" when converting +to SJIS for mobile phones. + +=item EUC-JP/JIS + +Converted to SJIS and then mapped to Unicode. Any non-SJIS character +in the string will not be mapped correctly. + +=item DoCoMo i-mode + +Portion of involving "EMOJI" in F800 - F9FF is maapped + to U+0FF800 - U+0FF9FF. + +=item ASTEL dot-i + +Portion of involving "EMOJI" in F000 - F4FF is mapped + to U+0FF000 - U+0FF4FF. + +=item J-PHONE J-SKY + +"J-SKY EMOJI" are mapped down as follows: "\e\$"(\x1b\x24) escape +sequences, the first byte, the second byte and "\x0f". +With sequential "EMOJI"s of identical first bytes, +it may be compressed by arranging only the second bytes. + +4500 - 47FF is mapped to U+0FFB00 - U+0FFDFF, accounting the first +and the second bytes make one EMOJI character. + +Unicode::Japanese will compress "J-SKY_EMOJI" automatically when +the first bytes of a sequence of "EMOJI" are identical. + +=back + +=head1 PurePerl mode + + use Unicode::Japanese qw(PurePerl); + +If module was loaded with 'PurePerl' keyword, +it works on Non-XS mode. + +=head1 BUGS + +=over 2 + +=item * + +EUC-JP, JIS strings cannot be converted correctly when they include +non-SJIS characters because they are converted to SJIS before +being converted to UTF-8. + +=item * + +Some characters of CP932 not in standard Shift_JIS +(ex; not in Joyo Kanji) will not be detected and converted. + +When string include such non-standard Shift_JIS, +they will not detected as SJIS. +Also, getcode() and all convert method will not work correctly. + +=item * + +When using XS, character encoding detection of EUC-JP and +SJIS(included all EMOJI) strings when they include "\e" will +fail. Also, getcode() and all convert method will not work. + +=item * + +The Japanese.pm file will collapse if sent via ASCII mode of FTP, +as it has a trailing binary data. + +=back + +=head1 AUTHOR INFORMATION + +Copyright 2001-2002 +SANO Taku (SAWATARI Mikage) and YAMASHINA Hio. +All right reserved. + +This library is free software; you can redistribute it +and/or modify it under the same terms as Perl itself. + +Bug reports and comments to: mikage@cpan.org. +Thank you. + +=head1 CREDITS + +Thanks very much to: + +NAKAYAMA Nao + +SUGIURA Tatsuki & Debian JP Project + +=cut + + + +__DATA__ + + + +%_h2zNum = ( + "0" => "\xef\xbc\x90", "1" => "\xef\xbc\x91", + "2" => "\xef\xbc\x92", "3" => "\xef\xbc\x93", + "4" => "\xef\xbc\x94", "5" => "\xef\xbc\x95", + "6" => "\xef\xbc\x96", "7" => "\xef\xbc\x97", + "8" => "\xef\xbc\x98", "9" => "\xef\xbc\x99", + +); + + + +%_z2hNum = ( + "\xef\xbc\x90" => "0", "\xef\xbc\x91" => "1", + "\xef\xbc\x92" => "2", "\xef\xbc\x93" => "3", + "\xef\xbc\x94" => "4", "\xef\xbc\x95" => "5", + "\xef\xbc\x96" => "6", "\xef\xbc\x97" => "7", + "\xef\xbc\x98" => "8", "\xef\xbc\x99" => "9", + +); + + + +%_h2zAlpha = ( + "A" => "\xef\xbc\xa1", "B" => "\xef\xbc\xa2", + "C" => "\xef\xbc\xa3", "D" => "\xef\xbc\xa4", + "E" => "\xef\xbc\xa5", "F" => "\xef\xbc\xa6", + "G" => "\xef\xbc\xa7", "H" => "\xef\xbc\xa8", + "I" => "\xef\xbc\xa9", "J" => "\xef\xbc\xaa", + "K" => "\xef\xbc\xab", "L" => "\xef\xbc\xac", + "M" => "\xef\xbc\xad", "N" => "\xef\xbc\xae", + "O" => "\xef\xbc\xaf", "P" => "\xef\xbc\xb0", + "Q" => "\xef\xbc\xb1", "R" => "\xef\xbc\xb2", + "S" => "\xef\xbc\xb3", "T" => "\xef\xbc\xb4", + "U" => "\xef\xbc\xb5", "V" => "\xef\xbc\xb6", + "W" => "\xef\xbc\xb7", "X" => "\xef\xbc\xb8", + "Y" => "\xef\xbc\xb9", "Z" => "\xef\xbc\xba", + "a" => "\xef\xbd\x81", "b" => "\xef\xbd\x82", + "c" => "\xef\xbd\x83", "d" => "\xef\xbd\x84", + "e" => "\xef\xbd\x85", "f" => "\xef\xbd\x86", + "g" => "\xef\xbd\x87", "h" => "\xef\xbd\x88", + "i" => "\xef\xbd\x89", "j" => "\xef\xbd\x8a", + "k" => "\xef\xbd\x8b", "l" => "\xef\xbd\x8c", + "m" => "\xef\xbd\x8d", "n" => "\xef\xbd\x8e", + "o" => "\xef\xbd\x8f", "p" => "\xef\xbd\x90", + "q" => "\xef\xbd\x91", "r" => "\xef\xbd\x92", + "s" => "\xef\xbd\x93", "t" => "\xef\xbd\x94", + "u" => "\xef\xbd\x95", "v" => "\xef\xbd\x96", + "w" => "\xef\xbd\x97", "x" => "\xef\xbd\x98", + "y" => "\xef\xbd\x99", "z" => "\xef\xbd\x9a", + +); + + + +%_z2hAlpha = ( + "\xef\xbc\xa1" => "A", "\xef\xbc\xa2" => "B", + "\xef\xbc\xa3" => "C", "\xef\xbc\xa4" => "D", + "\xef\xbc\xa5" => "E", "\xef\xbc\xa6" => "F", + "\xef\xbc\xa7" => "G", "\xef\xbc\xa8" => "H", + "\xef\xbc\xa9" => "I", "\xef\xbc\xaa" => "J", + "\xef\xbc\xab" => "K", "\xef\xbc\xac" => "L", + "\xef\xbc\xad" => "M", "\xef\xbc\xae" => "N", + "\xef\xbc\xaf" => "O", "\xef\xbc\xb0" => "P", + "\xef\xbc\xb1" => "Q", "\xef\xbc\xb2" => "R", + "\xef\xbc\xb3" => "S", "\xef\xbc\xb4" => "T", + "\xef\xbc\xb5" => "U", "\xef\xbc\xb6" => "V", + "\xef\xbc\xb7" => "W", "\xef\xbc\xb8" => "X", + "\xef\xbc\xb9" => "Y", "\xef\xbc\xba" => "Z", + "\xef\xbd\x81" => "a", "\xef\xbd\x82" => "b", + "\xef\xbd\x83" => "c", "\xef\xbd\x84" => "d", + "\xef\xbd\x85" => "e", "\xef\xbd\x86" => "f", + "\xef\xbd\x87" => "g", "\xef\xbd\x88" => "h", + "\xef\xbd\x89" => "i", "\xef\xbd\x8a" => "j", + "\xef\xbd\x8b" => "k", "\xef\xbd\x8c" => "l", + "\xef\xbd\x8d" => "m", "\xef\xbd\x8e" => "n", + "\xef\xbd\x8f" => "o", "\xef\xbd\x90" => "p", + "\xef\xbd\x91" => "q", "\xef\xbd\x92" => "r", + "\xef\xbd\x93" => "s", "\xef\xbd\x94" => "t", + "\xef\xbd\x95" => "u", "\xef\xbd\x96" => "v", + "\xef\xbd\x97" => "w", "\xef\xbd\x98" => "x", + "\xef\xbd\x99" => "y", "\xef\xbd\x9a" => "z", + +); + + + +%_h2zSym = ( + "\x20" => "\xe3\x80\x80", "\x21" => "\xef\xbc\x81", + "\x22" => "\xe2\x80\x9d", "\x23" => "\xef\xbc\x83", + "\x24" => "\xef\xbc\x84", "\x25" => "\xef\xbc\x85", + "\x26" => "\xef\xbc\x86", "\x27" => "\xef\xbf\xa5", + "\x28" => "\xef\xbc\x88", "\x29" => "\xef\xbc\x89", + "\x2a" => "\xef\xbc\x8a", "\x2b" => "\xef\xbc\x8b", + "\x2c" => "\xef\xbc\x8c", "\x2d" => "\xe2\x88\x92", + "\x2e" => "\xef\xbc\x8e", "\x2f" => "\xef\xbc\x8f", + "\x3a" => "\xef\xbc\x9a", "\x3b" => "\xef\xbc\x9b", + "\x3c" => "\xef\xbc\x9c", "\x3d" => "\xef\xbc\x9d", + "\x3e" => "\xef\xbc\x9e", "\x3f" => "\xef\xbc\x9f", + "\x40" => "\xef\xbc\xa0", "\x5b" => "\xef\xbc\xbb", + "\x5c" => "\xef\xbf\xa5", "\x5d" => "\xef\xbc\xbd", + "\x5e" => "\xef\xbc\xbe", "\x60" => "\xef\xbd\x80", + "\x7b" => "\xef\xbd\x9b", "\x7c" => "\xef\xbd\x9c", + "\x7d" => "\xef\xbd\x9d", "\x7e" => "\xe3\x80\x9c", + +); + + + +%_z2hSym = ( + "\xe3\x80\x80" => "\x20", "\xef\xbc\x8c" => "\x2c", + "\xef\xbc\x8e" => "\x2e", "\xef\xbc\x9a" => "\x3a", + "\xef\xbc\x9b" => "\x3b", "\xef\xbc\x9f" => "\x3f", + "\xef\xbc\x81" => "\x21", "\xef\xbd\x80" => "\x60", + "\xef\xbc\xbe" => "\x5e", "\xef\xbc\x8f" => "\x2f", + "\xe3\x80\x9c" => "\x7e", "\xef\xbd\x9c" => "\x7c", + "\xe2\x80\x9d" => "\x22", "\xef\xbc\x88" => "\x28", + "\xef\xbc\x89" => "\x29", "\xef\xbc\xbb" => "\x5b", + "\xef\xbc\xbd" => "\x5d", "\xef\xbd\x9b" => "\x7b", + "\xef\xbd\x9d" => "\x7d", "\xef\xbc\x8b" => "\x2b", + "\xe2\x88\x92" => "\x2d", "\xef\xbc\x9d" => "\x3d", + "\xef\xbc\x9c" => "\x3c", "\xef\xbc\x9e" => "\x3e", + "\xef\xbf\xa5" => "\x27", "\xef\xbc\x84" => "\x24", + "\xef\xbc\x85" => "\x25", "\xef\xbc\x83" => "\x23", + "\xef\xbc\x86" => "\x26", "\xef\xbc\x8a" => "\x2a", + "\xef\xbc\xa0" => "\x40", +); + + + +%_h2zKanaK = ( + "\xef\xbd\xa1" => "\xe3\x80\x82", "\xef\xbd\xa2" => "\xe3\x80\x8c", + "\xef\xbd\xa3" => "\xe3\x80\x8d", "\xef\xbd\xa4" => "\xe3\x80\x81", + "\xef\xbd\xa5" => "\xe3\x83\xbb", "\xef\xbd\xa6" => "\xe3\x83\xb2", + "\xef\xbd\xa7" => "\xe3\x82\xa1", "\xef\xbd\xa8" => "\xe3\x82\xa3", + "\xef\xbd\xa9" => "\xe3\x82\xa5", "\xef\xbd\xaa" => "\xe3\x82\xa7", + "\xef\xbd\xab" => "\xe3\x82\xa9", "\xef\xbd\xac" => "\xe3\x83\xa3", + "\xef\xbd\xad" => "\xe3\x83\xa5", "\xef\xbd\xae" => "\xe3\x83\xa7", + "\xef\xbd\xaf" => "\xe3\x83\x83", "\xef\xbd\xb0" => "\xe3\x83\xbc", + "\xef\xbd\xb1" => "\xe3\x82\xa2", "\xef\xbd\xb2" => "\xe3\x82\xa4", + "\xef\xbd\xb3" => "\xe3\x82\xa6", "\xef\xbd\xb4" => "\xe3\x82\xa8", + "\xef\xbd\xb5" => "\xe3\x82\xaa", "\xef\xbd\xb6" => "\xe3\x82\xab", + "\xef\xbd\xb7" => "\xe3\x82\xad", "\xef\xbd\xb8" => "\xe3\x82\xaf", + "\xef\xbd\xb9" => "\xe3\x82\xb1", "\xef\xbd\xba" => "\xe3\x82\xb3", + "\xef\xbd\xbb" => "\xe3\x82\xb5", "\xef\xbd\xbc" => "\xe3\x82\xb7", + "\xef\xbd\xbd" => "\xe3\x82\xb9", "\xef\xbd\xbe" => "\xe3\x82\xbb", + "\xef\xbd\xbf" => "\xe3\x82\xbd", "\xef\xbe\x80" => "\xe3\x82\xbf", + "\xef\xbe\x81" => "\xe3\x83\x81", "\xef\xbe\x82" => "\xe3\x83\x84", + "\xef\xbe\x83" => "\xe3\x83\x86", "\xef\xbe\x84" => "\xe3\x83\x88", + "\xef\xbe\x85" => "\xe3\x83\x8a", "\xef\xbe\x86" => "\xe3\x83\x8b", + "\xef\xbe\x87" => "\xe3\x83\x8c", "\xef\xbe\x88" => "\xe3\x83\x8d", + "\xef\xbe\x89" => "\xe3\x83\x8e", "\xef\xbe\x8a" => "\xe3\x83\x8f", + "\xef\xbe\x8b" => "\xe3\x83\x92", "\xef\xbe\x8c" => "\xe3\x83\x95", + "\xef\xbe\x8d" => "\xe3\x83\x98", "\xef\xbe\x8e" => "\xe3\x83\x9b", + "\xef\xbe\x8f" => "\xe3\x83\x9e", "\xef\xbe\x90" => "\xe3\x83\x9f", + "\xef\xbe\x91" => "\xe3\x83\xa0", "\xef\xbe\x92" => "\xe3\x83\xa1", + "\xef\xbe\x93" => "\xe3\x83\xa2", "\xef\xbe\x94" => "\xe3\x83\xa4", + "\xef\xbe\x95" => "\xe3\x83\xa6", "\xef\xbe\x96" => "\xe3\x83\xa8", + "\xef\xbe\x97" => "\xe3\x83\xa9", "\xef\xbe\x98" => "\xe3\x83\xaa", + "\xef\xbe\x99" => "\xe3\x83\xab", "\xef\xbe\x9a" => "\xe3\x83\xac", + "\xef\xbe\x9b" => "\xe3\x83\xad", "\xef\xbe\x9c" => "\xe3\x83\xaf", + "\xef\xbe\x9d" => "\xe3\x83\xb3", "\xef\xbe\x9e" => "\xe3\x82\x9b", + "\xef\xbe\x9f" => "\xe3\x82\x9c", +); + + + +%_z2hKanaK = ( + "\xe3\x80\x81" => "\xef\xbd\xa4", "\xe3\x80\x82" => "\xef\xbd\xa1", + "\xe3\x83\xbb" => "\xef\xbd\xa5", "\xe3\x82\x9b" => "\xef\xbe\x9e", + "\xe3\x82\x9c" => "\xef\xbe\x9f", "\xe3\x83\xbc" => "\xef\xbd\xb0", + "\xe3\x80\x8c" => "\xef\xbd\xa2", "\xe3\x80\x8d" => "\xef\xbd\xa3", + "\xe3\x82\xa1" => "\xef\xbd\xa7", "\xe3\x82\xa2" => "\xef\xbd\xb1", + "\xe3\x82\xa3" => "\xef\xbd\xa8", "\xe3\x82\xa4" => "\xef\xbd\xb2", + "\xe3\x82\xa5" => "\xef\xbd\xa9", "\xe3\x82\xa6" => "\xef\xbd\xb3", + "\xe3\x82\xa7" => "\xef\xbd\xaa", "\xe3\x82\xa8" => "\xef\xbd\xb4", + "\xe3\x82\xa9" => "\xef\xbd\xab", "\xe3\x82\xaa" => "\xef\xbd\xb5", + "\xe3\x82\xab" => "\xef\xbd\xb6", "\xe3\x82\xad" => "\xef\xbd\xb7", + "\xe3\x82\xaf" => "\xef\xbd\xb8", "\xe3\x82\xb1" => "\xef\xbd\xb9", + "\xe3\x82\xb3" => "\xef\xbd\xba", "\xe3\x82\xb5" => "\xef\xbd\xbb", + "\xe3\x82\xb7" => "\xef\xbd\xbc", "\xe3\x82\xb9" => "\xef\xbd\xbd", + "\xe3\x82\xbb" => "\xef\xbd\xbe", "\xe3\x82\xbd" => "\xef\xbd\xbf", + "\xe3\x82\xbf" => "\xef\xbe\x80", "\xe3\x83\x81" => "\xef\xbe\x81", + "\xe3\x83\x83" => "\xef\xbd\xaf", "\xe3\x83\x84" => "\xef\xbe\x82", + "\xe3\x83\x86" => "\xef\xbe\x83", "\xe3\x83\x88" => "\xef\xbe\x84", + "\xe3\x83\x8a" => "\xef\xbe\x85", "\xe3\x83\x8b" => "\xef\xbe\x86", + "\xe3\x83\x8c" => "\xef\xbe\x87", "\xe3\x83\x8d" => "\xef\xbe\x88", + "\xe3\x83\x8e" => "\xef\xbe\x89", "\xe3\x83\x8f" => "\xef\xbe\x8a", + "\xe3\x83\x92" => "\xef\xbe\x8b", "\xe3\x83\x95" => "\xef\xbe\x8c", + "\xe3\x83\x98" => "\xef\xbe\x8d", "\xe3\x83\x9b" => "\xef\xbe\x8e", + "\xe3\x83\x9e" => "\xef\xbe\x8f", "\xe3\x83\x9f" => "\xef\xbe\x90", + "\xe3\x83\xa0" => "\xef\xbe\x91", "\xe3\x83\xa1" => "\xef\xbe\x92", + "\xe3\x83\xa2" => "\xef\xbe\x93", "\xe3\x83\xa3" => "\xef\xbd\xac", + "\xe3\x83\xa4" => "\xef\xbe\x94", "\xe3\x83\xa5" => "\xef\xbd\xad", + "\xe3\x83\xa6" => "\xef\xbe\x95", "\xe3\x83\xa7" => "\xef\xbd\xae", + "\xe3\x83\xa8" => "\xef\xbe\x96", "\xe3\x83\xa9" => "\xef\xbe\x97", + "\xe3\x83\xaa" => "\xef\xbe\x98", "\xe3\x83\xab" => "\xef\xbe\x99", + "\xe3\x83\xac" => "\xef\xbe\x9a", "\xe3\x83\xad" => "\xef\xbe\x9b", + "\xe3\x83\xaf" => "\xef\xbe\x9c", "\xe3\x83\xb2" => "\xef\xbd\xa6", + "\xe3\x83\xb3" => "\xef\xbe\x9d", +); + + + +%_h2zKanaD = ( + "\xef\xbd\xb3\xef\xbe\x9e" => "\xe3\x83\xb4", "\xef\xbd\xb6\xef\xbe\x9e" => "\xe3\x82\xac", + "\xef\xbd\xb7\xef\xbe\x9e" => "\xe3\x82\xae", "\xef\xbd\xb8\xef\xbe\x9e" => "\xe3\x82\xb0", + "\xef\xbd\xb9\xef\xbe\x9e" => "\xe3\x82\xb2", "\xef\xbd\xba\xef\xbe\x9e" => "\xe3\x82\xb4", + "\xef\xbd\xbb\xef\xbe\x9e" => "\xe3\x82\xb6", "\xef\xbd\xbc\xef\xbe\x9e" => "\xe3\x82\xb8", + "\xef\xbd\xbd\xef\xbe\x9e" => "\xe3\x82\xba", "\xef\xbd\xbe\xef\xbe\x9e" => "\xe3\x82\xbc", + "\xef\xbd\xbf\xef\xbe\x9e" => "\xe3\x82\xbe", "\xef\xbe\x80\xef\xbe\x9e" => "\xe3\x83\x80", + "\xef\xbe\x81\xef\xbe\x9e" => "\xe3\x83\x82", "\xef\xbe\x82\xef\xbe\x9e" => "\xe3\x83\x85", + "\xef\xbe\x83\xef\xbe\x9e" => "\xe3\x83\x87", "\xef\xbe\x84\xef\xbe\x9e" => "\xe3\x83\x89", + "\xef\xbe\x8a\xef\xbe\x9e" => "\xe3\x83\x90", "\xef\xbe\x8a\xef\xbe\x9f" => "\xe3\x83\x91", + "\xef\xbe\x8b\xef\xbe\x9e" => "\xe3\x83\x93", "\xef\xbe\x8b\xef\xbe\x9f" => "\xe3\x83\x94", + "\xef\xbe\x8c\xef\xbe\x9e" => "\xe3\x83\x96", "\xef\xbe\x8c\xef\xbe\x9f" => "\xe3\x83\x97", + "\xef\xbe\x8d\xef\xbe\x9e" => "\xe3\x83\x99", "\xef\xbe\x8d\xef\xbe\x9f" => "\xe3\x83\x9a", + "\xef\xbe\x8e\xef\xbe\x9e" => "\xe3\x83\x9c", "\xef\xbe\x8e\xef\xbe\x9f" => "\xe3\x83\x9d", + +); + + + +%_z2hKanaD = ( + "\xe3\x82\xac" => "\xef\xbd\xb6\xef\xbe\x9e", "\xe3\x82\xae" => "\xef\xbd\xb7\xef\xbe\x9e", + "\xe3\x82\xb0" => "\xef\xbd\xb8\xef\xbe\x9e", "\xe3\x82\xb2" => "\xef\xbd\xb9\xef\xbe\x9e", + "\xe3\x82\xb4" => "\xef\xbd\xba\xef\xbe\x9e", "\xe3\x82\xb6" => "\xef\xbd\xbb\xef\xbe\x9e", + "\xe3\x82\xb8" => "\xef\xbd\xbc\xef\xbe\x9e", "\xe3\x82\xba" => "\xef\xbd\xbd\xef\xbe\x9e", + "\xe3\x82\xbc" => "\xef\xbd\xbe\xef\xbe\x9e", "\xe3\x82\xbe" => "\xef\xbd\xbf\xef\xbe\x9e", + "\xe3\x83\x80" => "\xef\xbe\x80\xef\xbe\x9e", "\xe3\x83\x82" => "\xef\xbe\x81\xef\xbe\x9e", + "\xe3\x83\x85" => "\xef\xbe\x82\xef\xbe\x9e", "\xe3\x83\x87" => "\xef\xbe\x83\xef\xbe\x9e", + "\xe3\x83\x89" => "\xef\xbe\x84\xef\xbe\x9e", "\xe3\x83\x90" => "\xef\xbe\x8a\xef\xbe\x9e", + "\xe3\x83\x91" => "\xef\xbe\x8a\xef\xbe\x9f", "\xe3\x83\x93" => "\xef\xbe\x8b\xef\xbe\x9e", + "\xe3\x83\x94" => "\xef\xbe\x8b\xef\xbe\x9f", "\xe3\x83\x96" => "\xef\xbe\x8c\xef\xbe\x9e", + "\xe3\x83\x97" => "\xef\xbe\x8c\xef\xbe\x9f", "\xe3\x83\x99" => "\xef\xbe\x8d\xef\xbe\x9e", + "\xe3\x83\x9a" => "\xef\xbe\x8d\xef\xbe\x9f", "\xe3\x83\x9c" => "\xef\xbe\x8e\xef\xbe\x9e", + "\xe3\x83\x9d" => "\xef\xbe\x8e\xef\xbe\x9f", "\xe3\x83\xb4" => "\xef\xbd\xb3\xef\xbe\x9e", + +); + + + +%_hira2kata = ( + "\xe3\x81\x81" => "\xe3\x82\xa1", "\xe3\x81\x82" => "\xe3\x82\xa2", + "\xe3\x81\x83" => "\xe3\x82\xa3", "\xe3\x81\x84" => "\xe3\x82\xa4", + "\xe3\x81\x85" => "\xe3\x82\xa5", "\xe3\x81\x86" => "\xe3\x82\xa6", + "\xe3\x81\x87" => "\xe3\x82\xa7", "\xe3\x81\x88" => "\xe3\x82\xa8", + "\xe3\x81\x89" => "\xe3\x82\xa9", "\xe3\x81\x8a" => "\xe3\x82\xaa", + "\xe3\x81\x8b" => "\xe3\x82\xab", "\xe3\x81\x8c" => "\xe3\x82\xac", + "\xe3\x81\x8d" => "\xe3\x82\xad", "\xe3\x81\x8e" => "\xe3\x82\xae", + "\xe3\x81\x8f" => "\xe3\x82\xaf", "\xe3\x81\x90" => "\xe3\x82\xb0", + "\xe3\x81\x91" => "\xe3\x82\xb1", "\xe3\x81\x92" => "\xe3\x82\xb2", + "\xe3\x81\x93" => "\xe3\x82\xb3", "\xe3\x81\x94" => "\xe3\x82\xb4", + "\xe3\x81\x95" => "\xe3\x82\xb5", "\xe3\x81\x96" => "\xe3\x82\xb6", + "\xe3\x81\x97" => "\xe3\x82\xb7", "\xe3\x81\x98" => "\xe3\x82\xb8", + "\xe3\x81\x99" => "\xe3\x82\xb9", "\xe3\x81\x9a" => "\xe3\x82\xba", + "\xe3\x81\x9b" => "\xe3\x82\xbb", "\xe3\x81\x9c" => "\xe3\x82\xbc", + "\xe3\x81\x9d" => "\xe3\x82\xbd", "\xe3\x81\x9e" => "\xe3\x82\xbe", + "\xe3\x81\x9f" => "\xe3\x82\xbf", "\xe3\x81\xa0" => "\xe3\x83\x80", + "\xe3\x81\xa1" => "\xe3\x83\x81", "\xe3\x81\xa2" => "\xe3\x83\x82", + "\xe3\x81\xa3" => "\xe3\x83\x83", "\xe3\x81\xa4" => "\xe3\x83\x84", + "\xe3\x81\xa5" => "\xe3\x83\x85", "\xe3\x81\xa6" => "\xe3\x83\x86", + "\xe3\x81\xa7" => "\xe3\x83\x87", "\xe3\x81\xa8" => "\xe3\x83\x88", + "\xe3\x81\xa9" => "\xe3\x83\x89", "\xe3\x81\xaa" => "\xe3\x83\x8a", + "\xe3\x81\xab" => "\xe3\x83\x8b", "\xe3\x81\xac" => "\xe3\x83\x8c", + "\xe3\x81\xad" => "\xe3\x83\x8d", "\xe3\x81\xae" => "\xe3\x83\x8e", + "\xe3\x81\xaf" => "\xe3\x83\x8f", "\xe3\x81\xb0" => "\xe3\x83\x90", + "\xe3\x81\xb1" => "\xe3\x83\x91", "\xe3\x81\xb2" => "\xe3\x83\x92", + "\xe3\x81\xb3" => "\xe3\x83\x93", "\xe3\x81\xb4" => "\xe3\x83\x94", + "\xe3\x81\xb5" => "\xe3\x83\x95", "\xe3\x81\xb6" => "\xe3\x83\x96", + "\xe3\x81\xb7" => "\xe3\x83\x97", "\xe3\x81\xb8" => "\xe3\x83\x98", + "\xe3\x81\xb9" => "\xe3\x83\x99", "\xe3\x81\xba" => "\xe3\x83\x9a", + "\xe3\x81\xbb" => "\xe3\x83\x9b", "\xe3\x81\xbc" => "\xe3\x83\x9c", + "\xe3\x81\xbd" => "\xe3\x83\x9d", "\xe3\x81\xbe" => "\xe3\x83\x9e", + "\xe3\x81\xbf" => "\xe3\x83\x9f", "\xe3\x82\x80" => "\xe3\x83\xa0", + "\xe3\x82\x81" => "\xe3\x83\xa1", "\xe3\x82\x82" => "\xe3\x83\xa2", + "\xe3\x82\x83" => "\xe3\x83\xa3", "\xe3\x82\x84" => "\xe3\x83\xa4", + "\xe3\x82\x85" => "\xe3\x83\xa5", "\xe3\x82\x86" => "\xe3\x83\xa6", + "\xe3\x82\x87" => "\xe3\x83\xa7", "\xe3\x82\x88" => "\xe3\x83\xa8", + "\xe3\x82\x89" => "\xe3\x83\xa9", "\xe3\x82\x8a" => "\xe3\x83\xaa", + "\xe3\x82\x8b" => "\xe3\x83\xab", "\xe3\x82\x8c" => "\xe3\x83\xac", + "\xe3\x82\x8d" => "\xe3\x83\xad", "\xe3\x82\x8e" => "\xe3\x83\xae", + "\xe3\x82\x8f" => "\xe3\x83\xaf", "\xe3\x82\x90" => "\xe3\x83\xb0", + "\xe3\x82\x91" => "\xe3\x83\xb1", "\xe3\x82\x92" => "\xe3\x83\xb2", + "\xe3\x82\x93" => "\xe3\x83\xb3", +); + + + +%_kata2hira = ( + "\xe3\x82\xa1" => "\xe3\x81\x81", "\xe3\x82\xa2" => "\xe3\x81\x82", + "\xe3\x82\xa3" => "\xe3\x81\x83", "\xe3\x82\xa4" => "\xe3\x81\x84", + "\xe3\x82\xa5" => "\xe3\x81\x85", "\xe3\x82\xa6" => "\xe3\x81\x86", + "\xe3\x82\xa7" => "\xe3\x81\x87", "\xe3\x82\xa8" => "\xe3\x81\x88", + "\xe3\x82\xa9" => "\xe3\x81\x89", "\xe3\x82\xaa" => "\xe3\x81\x8a", + "\xe3\x82\xab" => "\xe3\x81\x8b", "\xe3\x82\xac" => "\xe3\x81\x8c", + "\xe3\x82\xad" => "\xe3\x81\x8d", "\xe3\x82\xae" => "\xe3\x81\x8e", + "\xe3\x82\xaf" => "\xe3\x81\x8f", "\xe3\x82\xb0" => "\xe3\x81\x90", + "\xe3\x82\xb1" => "\xe3\x81\x91", "\xe3\x82\xb2" => "\xe3\x81\x92", + "\xe3\x82\xb3" => "\xe3\x81\x93", "\xe3\x82\xb4" => "\xe3\x81\x94", + "\xe3\x82\xb5" => "\xe3\x81\x95", "\xe3\x82\xb6" => "\xe3\x81\x96", + "\xe3\x82\xb7" => "\xe3\x81\x97", "\xe3\x82\xb8" => "\xe3\x81\x98", + "\xe3\x82\xb9" => "\xe3\x81\x99", "\xe3\x82\xba" => "\xe3\x81\x9a", + "\xe3\x82\xbb" => "\xe3\x81\x9b", "\xe3\x82\xbc" => "\xe3\x81\x9c", + "\xe3\x82\xbd" => "\xe3\x81\x9d", "\xe3\x82\xbe" => "\xe3\x81\x9e", + "\xe3\x82\xbf" => "\xe3\x81\x9f", "\xe3\x83\x80" => "\xe3\x81\xa0", + "\xe3\x83\x81" => "\xe3\x81\xa1", "\xe3\x83\x82" => "\xe3\x81\xa2", + "\xe3\x83\x83" => "\xe3\x81\xa3", "\xe3\x83\x84" => "\xe3\x81\xa4", + "\xe3\x83\x85" => "\xe3\x81\xa5", "\xe3\x83\x86" => "\xe3\x81\xa6", + "\xe3\x83\x87" => "\xe3\x81\xa7", "\xe3\x83\x88" => "\xe3\x81\xa8", + "\xe3\x83\x89" => "\xe3\x81\xa9", "\xe3\x83\x8a" => "\xe3\x81\xaa", + "\xe3\x83\x8b" => "\xe3\x81\xab", "\xe3\x83\x8c" => "\xe3\x81\xac", + "\xe3\x83\x8d" => "\xe3\x81\xad", "\xe3\x83\x8e" => "\xe3\x81\xae", + "\xe3\x83\x8f" => "\xe3\x81\xaf", "\xe3\x83\x90" => "\xe3\x81\xb0", + "\xe3\x83\x91" => "\xe3\x81\xb1", "\xe3\x83\x92" => "\xe3\x81\xb2", + "\xe3\x83\x93" => "\xe3\x81\xb3", "\xe3\x83\x94" => "\xe3\x81\xb4", + "\xe3\x83\x95" => "\xe3\x81\xb5", "\xe3\x83\x96" => "\xe3\x81\xb6", + "\xe3\x83\x97" => "\xe3\x81\xb7", "\xe3\x83\x98" => "\xe3\x81\xb8", + "\xe3\x83\x99" => "\xe3\x81\xb9", "\xe3\x83\x9a" => "\xe3\x81\xba", + "\xe3\x83\x9b" => "\xe3\x81\xbb", "\xe3\x83\x9c" => "\xe3\x81\xbc", + "\xe3\x83\x9d" => "\xe3\x81\xbd", "\xe3\x83\x9e" => "\xe3\x81\xbe", + "\xe3\x83\x9f" => "\xe3\x81\xbf", "\xe3\x83\xa0" => "\xe3\x82\x80", + "\xe3\x83\xa1" => "\xe3\x82\x81", "\xe3\x83\xa2" => "\xe3\x82\x82", + "\xe3\x83\xa3" => "\xe3\x82\x83", "\xe3\x83\xa4" => "\xe3\x82\x84", + "\xe3\x83\xa5" => "\xe3\x82\x85", "\xe3\x83\xa6" => "\xe3\x82\x86", + "\xe3\x83\xa7" => "\xe3\x82\x87", "\xe3\x83\xa8" => "\xe3\x82\x88", + "\xe3\x83\xa9" => "\xe3\x82\x89", "\xe3\x83\xaa" => "\xe3\x82\x8a", + "\xe3\x83\xab" => "\xe3\x82\x8b", "\xe3\x83\xac" => "\xe3\x82\x8c", + "\xe3\x83\xad" => "\xe3\x82\x8d", "\xe3\x83\xae" => "\xe3\x82\x8e", + "\xe3\x83\xaf" => "\xe3\x82\x8f", "\xe3\x83\xb0" => "\xe3\x82\x90", + "\xe3\x83\xb1" => "\xe3\x82\x91", "\xe3\x83\xb2" => "\xe3\x82\x92", + "\xe3\x83\xb3" => "\xe3\x82\x93", +); + + +} +sub h2zNum { + my $this = shift; + + if(!defined(%_h2zNum)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(0|1|2|3|4|5|6|7|8|9)/$_h2zNum{$1}/eg; + + $this; +} +sub euc +{ + my $this = shift; + $this->_s2e($this->sjis); +} +sub z2hKana +{ + my $this = shift; + + $this->z2hKanaD; + $this->z2hKanaK; + + $this; +} +sub splitCsv { + my $this = shift; + my $text = $this->{str}; + my @field; + + chomp($text); + + while ($text =~ m/"([^"\\]*(?:(?:\\.|\"\")[^"\\]*)*)",?|([^,]+),?|,/g) { + my $field = defined($1) ? $1 : (defined($2) ? $2 : ''); + $field =~ s/["\\]"/"/g; + push(@field, $field); + } + push(@field, '') if($text =~ m/,$/); + + \@field; + +} +sub strlen { + my $this = shift; + + my $ch_re = '[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}'; + my $length = 0; + + foreach my $c(split(/($ch_re)/,$this->{str})) { + next if(length($c) == 0); + $length += ((length($c) >= 3) ? 2 : 1); + } + + return $length; +} +sub join_csv { + my $this = shift; + + $this->joinCsv(@_); +} +sub utf16 +{ + my $this = shift; + $this->_utf8_utf16($this->{str}); +} +sub _utf16_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + my $sa; + foreach my $uc (unpack("n*", $str)) + { + ($uc >= 0xd800 and $uc <= 0xdbff and $sa = $uc and next); + + ($uc >= 0xdc00 and $uc <= 0xdfff and ($uc = ((($sa - 0xd800) << 10)|($uc - 0xdc00))+0x10000)); + + $result .= $U2T[$uc] ? $U2T[$uc] : + ($U2T[$uc] = ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x10000) ? chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xF0 | ($uc >> 18)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F))); + } + + $result; +} +sub _u2s { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : ( + $U2S{$1} + or ($U2S{$1} + = ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + # UTF-3バイト(U+0x80-U+07FF)からsjis-1バイトへのマッピングはないので\0を削除は必要はない + ($c eq "\0\0") ? '&#' . $ch . ';' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '&#' . $ch . ';' + ) + ), + ($c eq "\0\0") ? '&#' . $ch . ';' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + '?' + : '&#' . $ch . ';' + ) + ) : + (length($1) == 5) ? (($c1,$c2,$c3,$c4,$c5) = unpack("C5", $1), + $ch = (($c1 & 0x03) << 24)|(($c2 & 0x3F) << 18)| + (($c3 & 0x3f) << 12)|(($c4 & 0x3f) << 6)| + ($c5 & 0x3F), + '&#' . $ch . ';' + ) : + ( + ($c1,$c2,$c3,$c4,$c5,$c6) = unpack("C6", $1), + $ch = (($c1 & 0x03) << 30)|(($c2 & 0x3F) << 24)| + (($c3 & 0x3f) << 18)|(($c4 & 0x3f) << 12)| + (($c5 & 0x3f) << 6)|($c6 & 0x3F), + '&#' . $ch . ';' + ) + ) + ) + ) + /eg; + $str; + +} +sub _j2s2 { + my $this = shift; + my $esc = shift; + my $str = shift; + + if($esc eq $RE{JIS_0212}) + { + $str =~ s/../$CHARCODE{UNDEF_SJIS}/g; + } + elsif($esc !~ m/^$RE{JIS_ASC}/) + { + $str =~ tr/\x21-\x7e/\xa1-\xfe/; + if($esc =~ m/^$RE{JIS_0208}/) + { + $str =~ s/($RE{EUC_C})/ + $J2S[unpack('n', $1)] or $this->_j2s3($1) + /geo; + } + } + + $str; +} +sub z2hKanaD { + my $this = shift; + + if(!defined(%_z2hKanaD)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x82\xac|\xe3\x82\xae|\xe3\x82\xb0|\xe3\x82\xb2|\xe3\x82\xb4|\xe3\x82\xb6|\xe3\x82\xb8|\xe3\x82\xba|\xe3\x82\xbc|\xe3\x82\xbe|\xe3\x83\x80|\xe3\x83\x82|\xe3\x83\x85|\xe3\x83\x87|\xe3\x83\x89|\xe3\x83\x90|\xe3\x83\x91|\xe3\x83\x93|\xe3\x83\x94|\xe3\x83\x96|\xe3\x83\x97|\xe3\x83\x99|\xe3\x83\x9a|\xe3\x83\x9c|\xe3\x83\x9d|\xe3\x83\xb4)/$_z2hKanaD{$1}/eg; + + $this; +} +sub _j2s3 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if ($c1 % 2) + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x31 : 0x71); + $c2 -= 0x60 + ($c2 < 0xe0); + } + else + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x30 : 0x70); + $c2 -= 2; + } + + $J2S[unpack('n', $c)] = pack('CC', $c1, $c2); +} +sub joinCsv { + my $this = shift; + my $list; + + if(ref($_[0]) eq 'ARRAY') + { + $list = shift; + } + elsif(!ref($_[0])) + { + $list = [ @_ ]; + } + else + { + my $ref = ref($_[0]); + die "String->joinCsv, Param[1] is not ARRAY/ARRRAY-ref. [$ref]\n"; + } + + my $text = join ',', map {(s/"/""/g or /[\r\n,]/) ? qq("$_") : $_} @$list; + + $this->{str} = $text . "\n"; + + $this; +} +sub _utf32be_ucs4 { + my $this = shift; + my $str = shift; + + $str; +} +sub _s2e2 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if (0xa1 <= $c1 && $c1 <= 0xdf) + { + $c2 = $c1; + $c1 = 0x8e; + } + elsif (0x9f <= $c2) + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe0 : 0x60); + $c2 += 2; + } + else + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe1 : 0x61); + $c2 += 0x60 + ($c2 < 0x7f); + } + + $S2E[unpack('n', $c) or unpack('C', $1)] = pack('CC', $c1, $c2); +} +sub z2hKanaK { + my $this = shift; + + if(!defined(%_z2hKanaK)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x80\x81|\xe3\x80\x82|\xe3\x83\xbb|\xe3\x82\x9b|\xe3\x82\x9c|\xe3\x83\xbc|\xe3\x80\x8c|\xe3\x80\x8d|\xe3\x82\xa1|\xe3\x82\xa2|\xe3\x82\xa3|\xe3\x82\xa4|\xe3\x82\xa5|\xe3\x82\xa6|\xe3\x82\xa7|\xe3\x82\xa8|\xe3\x82\xa9|\xe3\x82\xaa|\xe3\x82\xab|\xe3\x82\xad|\xe3\x82\xaf|\xe3\x82\xb1|\xe3\x82\xb3|\xe3\x82\xb5|\xe3\x82\xb7|\xe3\x82\xb9|\xe3\x82\xbb|\xe3\x82\xbd|\xe3\x82\xbf|\xe3\x83\x81|\xe3\x83\x83|\xe3\x83\x84|\xe3\x83\x86|\xe3\x83\x88|\xe3\x83\x8a|\xe3\x83\x8b|\xe3\x83\x8c|\xe3\x83\x8d|\xe3\x83\x8e|\xe3\x83\x8f|\xe3\x83\x92|\xe3\x83\x95|\xe3\x83\x98|\xe3\x83\x9b|\xe3\x83\x9e|\xe3\x83\x9f|\xe3\x83\xa0|\xe3\x83\xa1|\xe3\x83\xa2|\xe3\x83\xa3|\xe3\x83\xa4|\xe3\x83\xa5|\xe3\x83\xa6|\xe3\x83\xa7|\xe3\x83\xa8|\xe3\x83\xa9|\xe3\x83\xaa|\xe3\x83\xab|\xe3\x83\xac|\xe3\x83\xad|\xe3\x83\xaf|\xe3\x83\xb2|\xe3\x83\xb3)/$_z2hKanaK{$1}/eg; + + $this; +} +sub h2zKana +{ + my $this = shift; + + $this->h2zKanaD; + $this->h2zKanaK; + + $this; +} +sub _ucs2_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + for my $uc (unpack("n*", $str)) + { + $result .= $U2T[$uc] ? $U2T[$uc] : + ($U2T[$uc] = ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . + chr(0x80 | ($uc & 0x3F))); + } + + $result; +} +sub z2hAlpha { + my $this = shift; + + if(!defined(%_z2hAlpha)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbc\xa1|\xef\xbc\xa2|\xef\xbc\xa3|\xef\xbc\xa4|\xef\xbc\xa5|\xef\xbc\xa6|\xef\xbc\xa7|\xef\xbc\xa8|\xef\xbc\xa9|\xef\xbc\xaa|\xef\xbc\xab|\xef\xbc\xac|\xef\xbc\xad|\xef\xbc\xae|\xef\xbc\xaf|\xef\xbc\xb0|\xef\xbc\xb1|\xef\xbc\xb2|\xef\xbc\xb3|\xef\xbc\xb4|\xef\xbc\xb5|\xef\xbc\xb6|\xef\xbc\xb7|\xef\xbc\xb8|\xef\xbc\xb9|\xef\xbc\xba|\xef\xbd\x81|\xef\xbd\x82|\xef\xbd\x83|\xef\xbd\x84|\xef\xbd\x85|\xef\xbd\x86|\xef\xbd\x87|\xef\xbd\x88|\xef\xbd\x89|\xef\xbd\x8a|\xef\xbd\x8b|\xef\xbd\x8c|\xef\xbd\x8d|\xef\xbd\x8e|\xef\xbd\x8f|\xef\xbd\x90|\xef\xbd\x91|\xef\xbd\x92|\xef\xbd\x93|\xef\xbd\x94|\xef\xbd\x95|\xef\xbd\x96|\xef\xbd\x97|\xef\xbd\x98|\xef\xbd\x99|\xef\xbd\x9a)/$_z2hAlpha{$1}/eg; + + $this; +} +sub _utf32le_ucs4 { + my $this = shift; + my $str = shift; + + my $result = ''; + foreach my $ch (unpack('V*', $str)) + { + $result .= pack('N', $ch); + } + + $result; +} +sub _utf8_utf16 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $uc; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})/ + $T2U{$1} + or ($T2U{$1} + = ((length($1) == 1) ? pack("n", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("n", (($c1 & 0x1F)<<6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("n", (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F))) : + (length($1) == 4) ? (($c1,$c2,$c3,$c4) = unpack("C4", $1), + ($uc = ((($c1 & 0x07) << 18)|(($c2 & 0x3F) << 12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F)) - 0x10000), + (($uc < 0x100000) ? pack("nn", (($uc >> 10) | 0xd800), (($uc & 0x3ff) | 0xdc00)) : "\0?")) : + "\0?") + ); + /eg; + $str; +} +sub getcode { + my $this = shift; + my $str = shift; + + my $l = length($str); + + if((($l % 4) == 0) + and ($str =~ m/^(?:$RE{BOM4_BE}|$RE{BOM4_LE})/o)) + { + return 'utf32'; + } + if((($l % 2) == 0) + and ($str =~ m/^(?:$RE{BOM2_BE}|$RE{BOM2_LE})/o)) + { + return 'utf16'; + } + + my $str2; + + if(($l % 4) == 0) + { + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{UTF32_BE})//o); + if($str2 eq '') + { + return 'utf32-be'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{UTF32_LE})//o); + if($str2 eq '') + { + return 'utf32-le'; + } + } + + if($str !~ m/[\e\x80-\xff]/) + { + return 'ascii'; + } + + if($str =~ m/$RE{JIS_0208}|$RE{JIS_0212}|$RE{JIS_ASC}|$RE{JIS_KANA}/o) + { + return 'jis'; + } + + if($str =~ m/(?:$RE{E_JSKY})/o) + { + return 'sjis-jsky'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{ASCII}|$RE{EUC_0212}|$RE{EUC_KANA}|$RE{EUC_C})//o); + if($str2 eq '') + { + return 'euc'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{ASCII}|$RE{SJIS_DBCS}|$RE{SJIS_KANA})//o); + if($str2 eq '') + { + return 'sjis'; + } + + my $str3; + $str3 = $str2; + 1 while($str3 =~ s/^(?:$RE{ASCII}|$RE{SJIS_DBCS}|$RE{SJIS_KANA}|$RE{E_IMODE})//o); + if($str3 eq '') + { + return 'sjis-imode'; + } + + $str3 = $str2; + 1 while($str3 =~ s/^(?:$RE{ASCII}|$RE{SJIS_DBCS}|$RE{SJIS_KANA}|$RE{E_DOTI})//o); + if($str3 eq '') + { + return 'sjis-doti'; + } + + $str2 = $str; + 1 while($str2 =~ s/^(?:$RE{UTF8})//o); + if($str2 eq '') + { + return 'utf8'; + } + + return 'unknown'; +} +sub _decodeBase64 +{ + local($^W) = 0; # unpack("u",...) gives bogus warning in 5.00[123] + + my $this = shift; + my $str = shift; + my $res = ""; + + $str =~ tr|A-Za-z0-9+=/||cd; # remove non-base64 chars + if (length($str) % 4) + { + warn("Length of base64 data not a multiple of 4"); + } + $str =~ s/=+$//; # remove padding + $str =~ tr|A-Za-z0-9+/| -_|; # convert to uuencoded format + while ($str =~ /(.{1,60})/gs) + { + my $len = chr(32 + length($1)*3/4); # compute length byte + $res .= unpack("u", $len . $1 ); # uudecode + } + $res; +} +sub sjis_doti +{ + my $this = shift; + $this->_u2sd($this->{str}); +} +sub sjis_jsky +{ + my $this = shift; + $this->_u2sj($this->{str}); +} +sub tag2bin { + my $this = shift; + + $this->{str} =~ s/\&(\#\d+|\#x[a-f0-9A-F]+);/ + (substr($1, 1, 1) eq 'x') ? $this->_ucs4_utf8(pack('N', hex(substr($1, 2)))) : + $this->_ucs4_utf8(pack('N', substr($1, 1))) + /eg; + + $this; +} +sub strcut +{ + my $this = shift; + my $cutlen = shift; + + if(ref($cutlen)) + { + die "String->strcut, Param[1] is Ref.\n"; + } + if($cutlen =~ m/\D/) + { + die "String->strcut, Param[1] must be NUMERIC.\n"; + } + + my $ch_re = '[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}'; + + my $result; + my $line = ''; + my $linelength = 0; + foreach my $c (split(/($ch_re)/, $this->{str})) + { + next if(length($c) == 0); + if($linelength + (length($c) >= 3 ? 2 : 1) > $cutlen) + { + push(@$result, $line); + $line = ''; + $linelength = 0; + } + $linelength += (length($c) >= 3 ? 2 : 1); + $line .= $c; + } + push(@$result, $line); + + $result; +} +sub h2zKanaD { + my $this = shift; + + if(!defined(%_h2zKanaD)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbd\xb3\xef\xbe\x9e|\xef\xbd\xb6\xef\xbe\x9e|\xef\xbd\xb7\xef\xbe\x9e|\xef\xbd\xb8\xef\xbe\x9e|\xef\xbd\xb9\xef\xbe\x9e|\xef\xbd\xba\xef\xbe\x9e|\xef\xbd\xbb\xef\xbe\x9e|\xef\xbd\xbc\xef\xbe\x9e|\xef\xbd\xbd\xef\xbe\x9e|\xef\xbd\xbe\xef\xbe\x9e|\xef\xbd\xbf\xef\xbe\x9e|\xef\xbe\x80\xef\xbe\x9e|\xef\xbe\x81\xef\xbe\x9e|\xef\xbe\x82\xef\xbe\x9e|\xef\xbe\x83\xef\xbe\x9e|\xef\xbe\x84\xef\xbe\x9e|\xef\xbe\x8a\xef\xbe\x9e|\xef\xbe\x8a\xef\xbe\x9f|\xef\xbe\x8b\xef\xbe\x9e|\xef\xbe\x8b\xef\xbe\x9f|\xef\xbe\x8c\xef\xbe\x9e|\xef\xbe\x8c\xef\xbe\x9f|\xef\xbe\x8d\xef\xbe\x9e|\xef\xbe\x8d\xef\xbe\x9f|\xef\xbe\x8e\xef\xbe\x9e|\xef\xbe\x8e\xef\xbe\x9f)/$_h2zKanaD{$1}/eg; + + $this; +} +sub _utf8_ucs2 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}|(.))/ + defined($2)?"\0$2": + $T2U{$1} + or ($T2U{$1} + = ((length($1) == 1) ? pack("n", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("n", (($c1 & 0x1F)<<6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("n", (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F))) : "\0?")) + /eg; + $str; +} +sub sjis_imode +{ + my $this = shift; + $this->_u2si($this->{str}); +} +sub _utf8_ucs4 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5}|(.))/ + defined($2) ? "\0\0\0$2" : + (length($1) == 1) ? pack("N", unpack("C", $1)) : + (length($1) == 2) ? (($c1,$c2) = unpack("C2", $1), + pack("N", (($c1 & 0x1F) << 6)|($c2 & 0x3F))) : + (length($1) == 3) ? (($c1,$c2,$c3) = unpack("C3", $1), + pack("N", (($c1 & 0x0F) << 12)|(($c2 & 0x3F) << 6)| + ($c3 & 0x3F))) : + (length($1) == 4) ? (($c1,$c2,$c3,$c4) = unpack("C4", $1), + pack("N", (($c1 & 0x07) << 18)|(($c2 & 0x3F) << 12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F))) : + (length($1) == 5) ? (($c1,$c2,$c3,$c4,$c5) = unpack("C5", $1), + pack("N", (($c1 & 0x03) << 24)|(($c2 & 0x3F) << 18)| + (($c3 & 0x3f) << 12)|(($c4 & 0x3f) << 6)| + ($c5 & 0x3F))) : + (($c1,$c2,$c3,$c4,$c5,$c6) = unpack("C6", $1), + pack("N", (($c1 & 0x03) << 30)|(($c2 & 0x3F) << 24)| + (($c3 & 0x3f) << 18)|(($c4 & 0x3f) << 12)| + (($c5 & 0x3f) << 6)|($c6 & 0x3F))) + /eg; + + $str; +} +sub _u2sd { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + if(!defined($eu2d)) + { + $eu2d = $this->_getFile('jcode/emoji/eu2d.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : + ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '?' + ) + ), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + ( + $c = substr($eu2d, ($ch - 0x0ff000) * 2, 2), + $c =~ tr,\0,,d, + ($c eq '') ? '?' : $c + ) : + '?' + ) + ) : + '?' + ) + /eg; + $str; + +} +sub get { + my $this = shift; + $this->{str}; +} +sub utf8 +{ + my $this = shift; + $this->{str}; +} +sub hira2kata { + my $this = shift; + + if(!defined(%_hira2kata)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x81\x81|\xe3\x81\x82|\xe3\x81\x83|\xe3\x81\x84|\xe3\x81\x85|\xe3\x81\x86|\xe3\x81\x87|\xe3\x81\x88|\xe3\x81\x89|\xe3\x81\x8a|\xe3\x81\x8b|\xe3\x81\x8c|\xe3\x81\x8d|\xe3\x81\x8e|\xe3\x81\x8f|\xe3\x81\x90|\xe3\x81\x91|\xe3\x81\x92|\xe3\x81\x93|\xe3\x81\x94|\xe3\x81\x95|\xe3\x81\x96|\xe3\x81\x97|\xe3\x81\x98|\xe3\x81\x99|\xe3\x81\x9a|\xe3\x81\x9b|\xe3\x81\x9c|\xe3\x81\x9d|\xe3\x81\x9e|\xe3\x81\x9f|\xe3\x81\xa0|\xe3\x81\xa1|\xe3\x81\xa2|\xe3\x81\xa3|\xe3\x81\xa4|\xe3\x81\xa5|\xe3\x81\xa6|\xe3\x81\xa7|\xe3\x81\xa8|\xe3\x81\xa9|\xe3\x81\xaa|\xe3\x81\xab|\xe3\x81\xac|\xe3\x81\xad|\xe3\x81\xae|\xe3\x81\xaf|\xe3\x81\xb0|\xe3\x81\xb1|\xe3\x81\xb2|\xe3\x81\xb3|\xe3\x81\xb4|\xe3\x81\xb5|\xe3\x81\xb6|\xe3\x81\xb7|\xe3\x81\xb8|\xe3\x81\xb9|\xe3\x81\xba|\xe3\x81\xbb|\xe3\x81\xbc|\xe3\x81\xbd|\xe3\x81\xbe|\xe3\x81\xbf|\xe3\x82\x80|\xe3\x82\x81|\xe3\x82\x82|\xe3\x82\x83|\xe3\x82\x84|\xe3\x82\x85|\xe3\x82\x86|\xe3\x82\x87|\xe3\x82\x88|\xe3\x82\x89|\xe3\x82\x8a|\xe3\x82\x8b|\xe3\x82\x8c|\xe3\x82\x8d|\xe3\x82\x8e|\xe3\x82\x8f|\xe3\x82\x90|\xe3\x82\x91|\xe3\x82\x92|\xe3\x82\x93)/$_hira2kata{$1}/eg; + + $this; +} +sub z2h { + my $this = shift; + + $this->z2hKana; + $this->z2hNum; + $this->z2hAlpha; + $this->z2hSym; + + $this; +} +sub _encodeBase64 +{ + my $this = shift; + my $str = shift; + my $eol = shift; + my $res = ""; + + $eol = "\n" unless defined $eol; + pos($str) = 0; # ensure start at the beginning + while ($str =~ /(.{1,45})/gs) + { + $res .= substr(pack('u', $1), 1); + chop($res); + } + $res =~ tr|` -_|AA-Za-z0-9+/|; # `# help emacs + # fix padding at the end + my $padding = (3 - length($str) % 3) % 3; + $res =~ s/.{$padding}$/'=' x $padding/e if $padding; + # break encoded string into lines of no more than 76 characters each + if (length $eol) + { + $res =~ s/(.{1,76})/$1$eol/g; + } + $res; +} +sub h2zKanaK { + my $this = shift; + + if(!defined(%_h2zKanaK)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbd\xa1|\xef\xbd\xa2|\xef\xbd\xa3|\xef\xbd\xa4|\xef\xbd\xa5|\xef\xbd\xa6|\xef\xbd\xa7|\xef\xbd\xa8|\xef\xbd\xa9|\xef\xbd\xaa|\xef\xbd\xab|\xef\xbd\xac|\xef\xbd\xad|\xef\xbd\xae|\xef\xbd\xaf|\xef\xbd\xb0|\xef\xbd\xb1|\xef\xbd\xb2|\xef\xbd\xb3|\xef\xbd\xb4|\xef\xbd\xb5|\xef\xbd\xb6|\xef\xbd\xb7|\xef\xbd\xb8|\xef\xbd\xb9|\xef\xbd\xba|\xef\xbd\xbb|\xef\xbd\xbc|\xef\xbd\xbd|\xef\xbd\xbe|\xef\xbd\xbf|\xef\xbe\x80|\xef\xbe\x81|\xef\xbe\x82|\xef\xbe\x83|\xef\xbe\x84|\xef\xbe\x85|\xef\xbe\x86|\xef\xbe\x87|\xef\xbe\x88|\xef\xbe\x89|\xef\xbe\x8a|\xef\xbe\x8b|\xef\xbe\x8c|\xef\xbe\x8d|\xef\xbe\x8e|\xef\xbe\x8f|\xef\xbe\x90|\xef\xbe\x91|\xef\xbe\x92|\xef\xbe\x93|\xef\xbe\x94|\xef\xbe\x95|\xef\xbe\x96|\xef\xbe\x97|\xef\xbe\x98|\xef\xbe\x99|\xef\xbe\x9a|\xef\xbe\x9b|\xef\xbe\x9c|\xef\xbe\x9d|\xef\xbe\x9e|\xef\xbe\x9f)/$_h2zKanaK{$1}/eg; + + $this; +} +sub _u2si { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + if(!defined($eu2i)) + { + $eu2i = $this->_getFile('jcode/emoji/eu2i.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : + ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '?' + ) + ), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + ( + $c = substr($eu2i, ($ch - 0x0ff000) * 2, 2), + $c =~ tr,\0,,d, + ($c eq '') ? '?' : $c + ) : + '?' + ) + ) : + '?' + ) + /eg; + $str; + +} +sub _u2sj { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($u2s_table)) + { + $u2s_table = $this->_getFile('jcode/u2s.dat'); + } + + if(!defined($eu2j)) + { + $eu2j = $this->_getFile('jcode/emoji/eu2j.dat'); + } + + my $c1; + my $c2; + my $c3; + my $c4; + my $c5; + my $c6; + my $c; + my $ch; + $str =~ s/([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3}|[\xf8-\xfb][\x80-\xbf]{4}|[\xfc-\xfd][\x80-\xbf]{5})|(.)/ + defined($2) ? '?' : + ((length($1) == 1) ? $1 : + (length($1) == 2) ? ( + ($c1,$c2) = unpack("C2", $1), + $ch = (($c1 & 0x1F)<<6)|($c2 & 0x3F), + $c = substr($u2s_table, $ch * 2, 2), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 3) ? ( + ($c1,$c2,$c3) = unpack("C3", $1), + $ch = (($c1 & 0x0F)<<12)|(($c2 & 0x3F)<<6)|($c3 & 0x3F), + ( + ($ch <= 0x9fff) ? + $c = substr($u2s_table, $ch * 2, 2) : + ($ch >= 0xf900 and $ch <= 0xffff) ? + ( + $c = substr($u2s_table, ($ch - 0xf900 + 0xa000) * 2, 2), + (($c =~ tr,\0,,d)==2 and $c = "\0\0"), + ) : + ( + $c = '?' + ) + ), + ($c eq "\0\0") ? '?' : $c + ) : + (length($1) == 4) ? ( + ($c1,$c2,$c3,$c4) = unpack("C4", $1), + $ch = (($c1 & 0x07)<<18)|(($c2 & 0x3F)<<12)| + (($c3 & 0x3f) << 6)|($c4 & 0x3F), + ( + ($ch >= 0x0ff000 and $ch <= 0x0fffff) ? + ( + $c = substr($eu2j, ($ch - 0x0ff000) * 5, 5), + $c =~ tr,\0,,d, + ($c eq '') ? '?' : $c + ) : + '?' + ) + ) : + '?' + ) + /eg; + + 1 while($str =~ s/($RE{E_JSKY_START})($RE{E_JSKY1})($RE{E_JSKY2}+)$RE{E_JSKY_END}$RE{E_JSKY_START}\2($RE{E_JSKY2})($RE{E_JSKY_END})/$1$2$3$4$5/o); + + $str; + +} +sub h2zAlpha { + my $this = shift; + + if(!defined(%_h2zAlpha)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z|a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)/$_h2zAlpha{$1}/eg; + + $this; +} +sub _s2e { + my $this = shift; + my $str = shift; + + $str =~ s/($RE{SJIS_DBCS}|$RE{SJIS_KANA})/ + $S2E[unpack('n', $1) or unpack('C', $1)] or $this->_s2e2($1) + /geo; + + $str; +} +sub _e2s2 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if ($c1 == 0x8e) + { # SS2 + $E2S[unpack('n', $c)] = chr($c2); + } + elsif ($c1 == 0x8f) + { # SS3 + $E2S[unpack('N', "\0" . $c)] = $CHARCODE{UNDEF_SJIS}; + } + else + { #SS1 or X0208 + if ($c1 % 2) + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x31 : 0x71); + $c2 -= 0x60 + ($c2 < 0xe0); + } + else + { + $c1 = ($c1>>1) + ($c1 < 0xdf ? 0x30 : 0x70); + $c2 -= 2; + } + $E2S[unpack('n', $c)] = pack('CC', $c1, $c2); + } +} +sub _utf16le_utf16 { + my $this = shift; + my $str = shift; + + my $result = ''; + foreach my $ch (unpack('v*', $str)) + { + $result .= pack('n', $ch); + } + + $result; +} +sub _sj2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + if(!defined($ej2u)) + { + $ej2u = $this->_getFile('jcode/emoji/ej2u.dat'); + } + + my $l; + my $j1; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|$RE{E_JSKY}|[\x00-\xff])/ + (length($1) <= 2) ? + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xffff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) : + ( + $l = $1, + $l =~ s,^$RE{E_JSKY_START}($RE{E_JSKY1}),,o, + $j1 = $1, + $uc = '', + $l =~ s!($RE{E_JSKY2})!$uc .= substr($ej2u, (unpack('n', $j1 . $1) - 0x4500) * 4, 4), ''!ego, + $uc =~ tr,\0,,d, + $uc + ) + /eg; + + $str; + +} +sub _s2j { + my $this = shift; + my $str = shift; + + $str =~ s/((?:$RE{SJIS_DBCS}|$RE{SJIS_KANA})+)/ + $this->_s2j2($1) . $ESC{ASC} + /geo; + + $str; +} +sub _s2j2 { + my $this = shift; + my $str = shift; + + $str =~ s/((?:$RE{SJIS_DBCS})+|(?:$RE{SJIS_KANA})+)/ + my $s = $1; + if($s =~ m,^$RE{SJIS_KANA},) + { + $s =~ tr,\xa1-\xdf,\x21-\x5f,; + $ESC{KANA} . $s + } + else + { + $s =~ s!($RE{SJIS_DBCS})! + $S2J[unpack('n', $1)] or $this->_s2j3($1) + !geo; + $ESC{JIS_0208} . $s; + } + /geo; + + $str; +} +sub _s2j3 { + my $this = shift; + my $c = shift; + + my ($c1, $c2) = unpack('CC', $c); + if (0x9f <= $c2) + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe0 : 0x60); + $c2 += 2; + } + else + { + $c1 = $c1 * 2 - ($c1 >= 0xe0 ? 0xe1 : 0x61); + $c2 += 0x60 + ($c2 < 0x7f); + } + + $S2J[unpack('n', $c)] = pack('CC', $c1 - 0x80, $c2 - 0x80); +} +sub conv { + my $this = shift; + my $ocode = shift; + my $encode = shift; + my (@option) = @_; + + my $res; + if($ocode eq 'utf8') + { + $res = $this->utf8; + } + elsif($ocode eq 'euc') + { + $res = $this->euc; + } + elsif($ocode eq 'jis') + { + $res = $this->jis; + } + elsif($ocode eq 'sjis') + { + $res = $this->sjis; + } + elsif($ocode eq 'sjis-imode') + { + $res = $this->sjis_imode; + } + elsif($ocode eq 'sjis-doti') + { + $res = $this->sjis_doti; + } + elsif($ocode eq 'sjis-jsky') + { + $res = $this->sjis_jsky; + } + elsif($ocode eq 'ucs2') + { + $res = $this->ucs2; + } + elsif($ocode eq 'ucs4') + { + $res = $this->ucs4; + } + elsif($ocode eq 'utf16') + { + $res = $this->utf16; + } + elsif($ocode eq 'binary') + { + $res = $this->{str}; + } + else + { + die qq(String->conv, Param[1] "$ocode" is error.\n); + } + + if(defined($encode)) + { + if($encode eq 'base64') + { + $res = $this->_encodeBase64($res, @option); + } + else + { + die qq(String->conv, Param[2] "$encode" encode name error.\n); + } + } + + $res; +} +sub _s2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + my $l; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|[\x00-\xff])/ + $S2U{$1} + or ($S2U{$1} = + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xfcff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) + )/eg; + + $str; + +} +sub _j2s { + my $this = shift; + my $str = shift; + + $str =~ s/($RE{JIS_0208}|$RE{JIS_0212}|$RE{JIS_ASC}|$RE{JIS_KANA})([^\e]*)/ + $this->_j2s2($1, $2) + /geo; + + $str; +} +sub h2z { + my $this = shift; + + $this->h2zKana; + $this->h2zNum; + $this->h2zAlpha; + $this->h2zSym; + + $this; +} +sub ucs2 +{ + my $this = shift; + $this->_utf8_ucs2($this->{str}); +} +sub z2hSym { + my $this = shift; + + if(!defined(%_z2hSym)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x80\x80|\xef\xbc\x8c|\xef\xbc\x8e|\xef\xbc\x9a|\xef\xbc\x9b|\xef\xbc\x9f|\xef\xbc\x81|\xef\xbd\x80|\xef\xbc\xbe|\xef\xbc\x8f|\xe3\x80\x9c|\xef\xbd\x9c|\xe2\x80\x9d|\xef\xbc\x88|\xef\xbc\x89|\xef\xbc\xbb|\xef\xbc\xbd|\xef\xbd\x9b|\xef\xbd\x9d|\xef\xbc\x8b|\xe2\x88\x92|\xef\xbc\x9d|\xef\xbc\x9c|\xef\xbc\x9e|\xef\xbf\xa5|\xef\xbc\x84|\xef\xbc\x85|\xef\xbc\x83|\xef\xbc\x86|\xef\xbc\x8a|\xef\xbc\xa0)/$_z2hSym{$1}/eg; + + $this; +} +sub set +{ + my $this = shift; + my $str = shift; + my $icode = shift; + my $encode = shift; + + if(ref($str)) + { + die "String->set, Param[1] is Ref.\n"; + } + if(ref($icode)) + { + die "String->set, Param[2] is Ref.\n"; + } + if(ref($encode)) + { + die "String->set, Param[3] is Ref.\n"; + } + + if(defined($encode)) + { + if($encode eq 'base64') + { + $str = $this->_decodeBase64($str); + } + else + { + die "String->set, Param[3] encode name error.\n"; + } + } + + if(!defined($icode)) + { + $this->{str} = $str; + } + else + { + $icode = lc($icode); + if($icode eq 'auto') + { + $icode = $this->getcode($str); + } + if($icode eq 'utf8') + { + $this->{str} = $str; + } + elsif($icode eq 'ucs2') + { + $this->{str} = $this->_ucs2_utf8($str); + } + elsif($icode eq 'ucs4') + { + $this->{str} = $this->_ucs4_utf8($str); + } + elsif($icode eq 'utf16-be') + { + $this->{str} = $this->_utf16_utf8($this->_utf16be_utf16($str)); + } + elsif($icode eq 'utf16-le') + { + $this->{str} = $this->_utf16_utf8($this->_utf16le_utf16($str)); + } + elsif($icode eq 'utf16') + { + $this->{str} = $this->_utf16_utf8($this->_utf16_utf16($str)); + } + elsif($icode eq 'utf32-be') + { + $this->{str} = $this->_ucs4_utf8($this->_utf32be_ucs4($str)); + } + elsif($icode eq 'utf32-le') + { + $this->{str} = $this->_ucs4_utf8($this->_utf32le_ucs4($str)); + } + elsif($icode eq 'utf32') + { + $this->{str} = $this->_ucs4_utf8($this->_utf32_ucs4($str)); + } + elsif($icode eq 'jis') + { + $this->{str} = $this->_j2s($str); + $this->{str} = $this->_s2u($this->{str}); + } + elsif($icode eq 'euc') + { + $this->{str} = $this->_e2s($str); + $this->{str} = $this->_s2u($this->{str}); + } + elsif($icode eq 'sjis') + { + $this->{str} = $this->_s2u($str); + } + elsif($icode eq 'sjis-imode') + { + $this->{str} = $this->_si2u($str); + } + elsif($icode eq 'sjis-doti') + { + $this->{str} = $this->_sd2u($str); + } + elsif($icode eq 'sjis-jsky') + { + $this->{str} = $this->_sj2u($str); + } + elsif($icode eq 'ascii') + { + $this->{str} = $str; + } + elsif($icode eq 'unknown') + { + $this->{str} = $str; + } + elsif($icode eq 'binary') + { + $this->{str} = $str; + } + else + { + use Carp; + croak "icode error [$icode]"; + } + } + + $this; +} +sub ucs4 +{ + my $this = shift; + $this->_utf8_ucs4($this->{str}); +} +sub z2hNum { + my $this = shift; + + if(!defined(%_z2hNum)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xef\xbc\x90|\xef\xbc\x91|\xef\xbc\x92|\xef\xbc\x93|\xef\xbc\x94|\xef\xbc\x95|\xef\xbc\x96|\xef\xbc\x97|\xef\xbc\x98|\xef\xbc\x99)/$_z2hNum{$1}/eg; + + $this; +} +sub _si2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + if(!defined($ei2u)) + { + $ei2u = $this->_getFile('jcode/emoji/ei2u.dat'); + } + + $str =~ s/(\&\#(\d+);)/ + ($2 >= 0xf800 and $2 <= 0xf9ff) ? pack('n', $2) : $1 + /eg; + + my $l; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|$RE{E_IMODE}|[\x00-\xff])/ + $S2U{$1} + or ($S2U{$1} = + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xf800 and $l <= 0xf9ff) ? + ( + $uc = substr($ei2u, ($l - 0xf800) * 4, 4), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xffff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) + )/eg; + + $str; + +} +sub _e2s { + my $this = shift; + my $str = shift; + + $str =~ s/($RE{EUC_KANA}|$RE{EUC_0212}|$RE{EUC_C})/ + $E2S[unpack('n', $1) or unpack('N', "\0" . $1)] or $this->_e2s2($1) + /geo; + + $str; +} +sub jis +{ + my $this = shift; + $this->_s2j($this->sjis); +} +sub _utf32_ucs4 { + my $this = shift; + my $str = shift; + + if($str =~ s/^\x00\x00\xfe\xff//) + { + $str = $this->_utf32be_ucs4($str); + } + elsif($str =~ s/^\xff\xfe\x00\x00//) + { + $str = $this->_utf32le_ucs4($str); + } + else + { + $str = $this->_utf32be_ucs4($str); + } + + $str; +} +sub kata2hira { + my $this = shift; + + if(!defined(%_kata2hira)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\xe3\x82\xa1|\xe3\x82\xa2|\xe3\x82\xa3|\xe3\x82\xa4|\xe3\x82\xa5|\xe3\x82\xa6|\xe3\x82\xa7|\xe3\x82\xa8|\xe3\x82\xa9|\xe3\x82\xaa|\xe3\x82\xab|\xe3\x82\xac|\xe3\x82\xad|\xe3\x82\xae|\xe3\x82\xaf|\xe3\x82\xb0|\xe3\x82\xb1|\xe3\x82\xb2|\xe3\x82\xb3|\xe3\x82\xb4|\xe3\x82\xb5|\xe3\x82\xb6|\xe3\x82\xb7|\xe3\x82\xb8|\xe3\x82\xb9|\xe3\x82\xba|\xe3\x82\xbb|\xe3\x82\xbc|\xe3\x82\xbd|\xe3\x82\xbe|\xe3\x82\xbf|\xe3\x83\x80|\xe3\x83\x81|\xe3\x83\x82|\xe3\x83\x83|\xe3\x83\x84|\xe3\x83\x85|\xe3\x83\x86|\xe3\x83\x87|\xe3\x83\x88|\xe3\x83\x89|\xe3\x83\x8a|\xe3\x83\x8b|\xe3\x83\x8c|\xe3\x83\x8d|\xe3\x83\x8e|\xe3\x83\x8f|\xe3\x83\x90|\xe3\x83\x91|\xe3\x83\x92|\xe3\x83\x93|\xe3\x83\x94|\xe3\x83\x95|\xe3\x83\x96|\xe3\x83\x97|\xe3\x83\x98|\xe3\x83\x99|\xe3\x83\x9a|\xe3\x83\x9b|\xe3\x83\x9c|\xe3\x83\x9d|\xe3\x83\x9e|\xe3\x83\x9f|\xe3\x83\xa0|\xe3\x83\xa1|\xe3\x83\xa2|\xe3\x83\xa3|\xe3\x83\xa4|\xe3\x83\xa5|\xe3\x83\xa6|\xe3\x83\xa7|\xe3\x83\xa8|\xe3\x83\xa9|\xe3\x83\xaa|\xe3\x83\xab|\xe3\x83\xac|\xe3\x83\xad|\xe3\x83\xae|\xe3\x83\xaf|\xe3\x83\xb0|\xe3\x83\xb1|\xe3\x83\xb2|\xe3\x83\xb3)/$_kata2hira{$1}/eg; + + $this; +} +sub _ucs4_utf8 { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + my $result = ''; + for my $uc (unpack("N*", $str)) + { + $result .= ($uc < 0x80) ? chr($uc) : + ($uc < 0x800) ? chr(0xC0 | ($uc >> 6)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x10000) ? chr(0xE0 | ($uc >> 12)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x200000) ? chr(0xF0 | ($uc >> 18)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + ($uc < 0x4000000) ? chr(0xF8 | ($uc >> 24)) . chr(0x80 | (($uc >> 18) & 0x3F)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)) : + chr(0xFC | ($uc >> 30)) . chr(0x80 | (($uc >> 24) & 0x3F)) . chr(0x80 | (($uc >> 18) & 0x3F)) . chr(0x80 | (($uc >> 12) & 0x3F)) . chr(0x80 | (($uc >> 6) & 0x3F)) . chr(0x80 | ($uc & 0x3F)); + } + + $result; +} +sub split_csv { + my $this = shift; + + $this->splitCsv(@_); +} +sub _utf16_utf16 { + my $this = shift; + my $str = shift; + + if($str =~ s/^\xfe\xff//) + { + $str = $this->_utf16be_utf16($str); + } + elsif($str =~ s/^\xff\xfe//) + { + $str = $this->_utf16le_utf16($str); + } + else + { + $str = $this->_utf16be_utf16($str); + } + + $str; +} +sub _sd2u { + my $this = shift; + my $str = shift; + + if(!defined($str)) + { + return ''; + } + + if(!defined($s2u_table)) + { + $s2u_table = $this->_getFile('jcode/s2u.dat'); + } + + if(!defined($ed2u)) + { + $ed2u = $this->_getFile('jcode/emoji/ed2u.dat'); + } + + $str =~ s/(\&\#(\d+);)/ + ($2 >= 0xf000 and $2 <= 0xf4ff) ? pack('n', $2) : $1 + /eg; + + my $l; + my $uc; + $str =~ s/($RE{SJIS_KANA}|$RE{SJIS_DBCS}|$RE{E_DOTI}|[\x00-\xff])/ + $S2U{$1} + or ($S2U{$1} = + ( + $l = (unpack('n', $1) or unpack('C', $1)), + ( + ($l >= 0xa1 and $l <= 0xdf) ? + ( + $uc = substr($s2u_table, ($l - 0xa1) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0x8100 and $l <= 0x9fff) ? + ( + $uc = substr($s2u_table, ($l - 0x8100 + 0x3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xf000 and $l <= 0xf4ff) ? + ( + $uc = substr($ed2u, ($l - 0xf000) * 4, 4), + $uc =~ tr,\0,,d, + $uc + ) : + ($l >= 0xe000 and $l <= 0xffff) ? + ( + $uc = substr($s2u_table, ($l - 0xe000 + 0x1f3f) * 3, 3), + $uc =~ tr,\0,,d, + $uc + ) : + ($l < 0x80) ? + chr($l) : + '?' + ) + ) + )/eg; + + $str; + +} +sub sjis +{ + my $this = shift; + $this->_u2s($this->{str}); +} +sub _utf16be_utf16 { + my $this = shift; + my $str = shift; + + $str; +} +sub h2zSym { + my $this = shift; + + if(!defined(%_h2zSym)) + { + $this->_loadConvTable; + } + + $this->{str} =~ s/(\x20|\x21|\x22|\x23|\x24|\x25|\x26|\x27|\x28|\x29|\x2a|\x2b|\x2c|\x2d|\x2e|\x2f|\x3a|\x3b|\x3c|\x3d|\x3e|\x3f|\x40|\x5b|\x5c|\x5d|\x5e|\x60|\x7b|\x7c|\x7d|\x7e)/$_h2zSym{$1}/eg; + + $this; +} + + |
