Skip to content

Commit

Permalink
styles fiexed 💄
Browse files Browse the repository at this point in the history
  • Loading branch information
RahulDey12 committed Jun 9, 2024
1 parent 8b92f78 commit 1e57e71
Show file tree
Hide file tree
Showing 23 changed files with 94 additions and 124 deletions.
10 changes: 5 additions & 5 deletions src/Bpe.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ final class Bpe implements BpeContract
private readonly string $specialRegex;

/**
* @param Vocab $vocab
* @param array<string, int> $specialTokens
* @param string $regex
* @param array<string, int> $specialTokens
*
* @throws Exceptions\InvalidPatternException
*/
public function __construct(
Expand All @@ -31,7 +30,7 @@ public function __construct(
}

/**
* @inheritDoc
* {@inheritDoc}
*/
public function encode(string $text, array $allowedSpecial): array
{
Expand Down Expand Up @@ -98,7 +97,8 @@ public function encode(string $text, array $allowedSpecial): array
}

/**
* @inheritDoc
* {@inheritDoc}
*
* @throws Exception
*/
public function encodeOrdinary(string $text): array
Expand Down
4 changes: 1 addition & 3 deletions src/Contracts/BpeContract.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@
interface BpeContract
{
/**
* @param string $text
* @param string[] $allowedSpecial
* @param string[] $allowedSpecial
* @return array{0: int[], 1, int}
*/
public function encode(string $text, array $allowedSpecial): array;

/**
* @param string $text
* @return int[]
*/
public function encodeOrdinary(string $text): array;
Expand Down
40 changes: 18 additions & 22 deletions src/Encoder.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,7 @@ class Encoder
protected int $maxTokenValue;

/**
* @param string $name
* @param string $pattern
* @param Vocab $vocab
* @param array<string, int> $specialTokens
* @param int|null $vocabLength
* @param BpeContract|null $bpe
* @param array<string, int> $specialTokens
*/
public function __construct(
public readonly string $name,
Expand All @@ -29,14 +24,14 @@ public function __construct(
public readonly ?int $vocabLength = null,
protected ?BpeContract $bpe = null,
) {
if(is_null($this->bpe)) {
if (is_null($this->bpe)) {
$this->initializeBpe();
}
}

/**
* @param string $text
* @return int[]
*
* @throws \Exception
*/
public function encodeOrdinary(string $text): array
Expand All @@ -45,8 +40,9 @@ public function encodeOrdinary(string $text): array
}

/**
* @param string[] $texts
* @param string[] $texts
* @return array<int[]>
*
* @throws \Exception
*/
public function encodeOrdinaryBatch(array $texts): array
Expand All @@ -61,10 +57,9 @@ public function encodeOrdinaryBatch(array $texts): array
}

/**
* @param string $text
* @param string[]|'all' $allowedSpecial
* @param string $disallowedSpecial
* @param string[]|'all' $allowedSpecial
* @return int[]
*
* @throws Exceptions\InvalidPatternException
* @throws SpecialTokenNotAllowedException
*/
Expand All @@ -91,10 +86,10 @@ public function encode(string $text, array|string $allowedSpecial = [], string $
}

/**
* @param array<string> $texts
* @param string[]|'all' $allowedSpecial
* @param string $disallowedSpecial
* @param array<string> $texts
* @param string[]|'all' $allowedSpecial
* @return array<int[]>
*
* @throws Exceptions\InvalidPatternException
* @throws SpecialTokenNotAllowedException
*/
Expand All @@ -110,21 +105,21 @@ public function encodeBatch(array $texts, array|string $allowedSpecial = [], str
}

/**
* @param int[] $tokens
* @return string
* @param int[] $tokens
*
* @throws RankNotFoundException
*/
public function decode(array $tokens): string
{
$text = '';

foreach ($tokens as $token) {
try{
try {
$text .= $this->vocab->getToken($token);
}catch (RankNotFoundException $exception){
} catch (RankNotFoundException $exception) {
$piece = array_search($token, $this->specialTokens);

if(! $piece) {
if (! $piece) {
throw $exception;
}

Expand All @@ -136,8 +131,9 @@ public function decode(array $tokens): string
}

/**
* @param array<int[]> $batch
* @param array<int[]> $batch
* @return array<string>
*
* @throws RankNotFoundException
*/
public function decodeBatch(array $batch): array
Expand Down Expand Up @@ -183,7 +179,7 @@ public function setBpe(Bpe $bpe): void

public function getBpe(): BpeContract
{
if(is_null($this->bpe)) {
if (is_null($this->bpe)) {
throw new \Exception('Bpe Not Found');
}

Expand Down
2 changes: 0 additions & 2 deletions src/Encodings/OpenAiPublic/AbstractEncoding.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

use Rahul900day\Tiktoken\Contracts\EncodingContract;
use Rahul900day\Tiktoken\Encoder;
use Rahul900day\Tiktoken\Enums\SpecialToken;
use Rahul900day\Tiktoken\Vocab;

abstract class AbstractEncoding implements EncodingContract
Expand Down Expand Up @@ -34,5 +33,4 @@ public function __invoke(): Encoder
$this->vocabLength,
);
}

}
2 changes: 1 addition & 1 deletion src/Encodings/OpenAiPublic/Cl100KBaseEncoding.php
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ protected function getPattern(): string
}

/**
* @inheritDoc
* {@inheritDoc}
*/
protected function getSpecialTokens(): array
{
Expand Down
4 changes: 1 addition & 3 deletions src/Encodings/OpenAiPublic/Gpt2Encoding.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

namespace Rahul900day\Tiktoken\Encodings\OpenAiPublic;

use Rahul900day\Tiktoken\Contracts\EncodingContract;
use Rahul900day\Tiktoken\Encoder;
use Rahul900day\Tiktoken\Enums\SpecialToken;
use Rahul900day\Tiktoken\Loaders\DataGymLoader;
use Rahul900day\Tiktoken\Readers\HttpReader;
Expand Down Expand Up @@ -33,7 +31,7 @@ protected function getPattern(): string
}

/**
* @inheritDoc
* {@inheritDoc}
*/
protected function getSpecialTokens(): array
{
Expand Down
4 changes: 1 addition & 3 deletions src/Encodings/OpenAiPublic/P50KBaseEncoding.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

namespace Rahul900day\Tiktoken\Encodings\OpenAiPublic;

use Rahul900day\Tiktoken\Contracts\EncodingContract;
use Rahul900day\Tiktoken\Encoder;
use Rahul900day\Tiktoken\Enums\SpecialToken;
use Rahul900day\Tiktoken\Loaders\TiktokenLoader;
use Rahul900day\Tiktoken\Readers\HttpReader;
Expand Down Expand Up @@ -33,7 +31,7 @@ protected function getPattern(): string
}

/**
* @inheritDoc
* {@inheritDoc}
*/
protected function getSpecialTokens(): array
{
Expand Down
4 changes: 1 addition & 3 deletions src/Encodings/OpenAiPublic/P50KEditEncoding.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

namespace Rahul900day\Tiktoken\Encodings\OpenAiPublic;

use Rahul900day\Tiktoken\Contracts\EncodingContract;
use Rahul900day\Tiktoken\Encoder;
use Rahul900day\Tiktoken\Enums\SpecialToken;
use Rahul900day\Tiktoken\Loaders\TiktokenLoader;
use Rahul900day\Tiktoken\Readers\HttpReader;
Expand All @@ -31,7 +29,7 @@ protected function getPattern(): string
}

/**
* @inheritDoc
* {@inheritDoc}
*/
protected function getSpecialTokens(): array
{
Expand Down
4 changes: 1 addition & 3 deletions src/Encodings/OpenAiPublic/R50KBaseEncoding.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

namespace Rahul900day\Tiktoken\Encodings\OpenAiPublic;

use Rahul900day\Tiktoken\Contracts\EncodingContract;
use Rahul900day\Tiktoken\Encoder;
use Rahul900day\Tiktoken\Enums\SpecialToken;
use Rahul900day\Tiktoken\Loaders\TiktokenLoader;
use Rahul900day\Tiktoken\Readers\HttpReader;
Expand Down Expand Up @@ -33,7 +31,7 @@ protected function getPattern(): string
}

/**
* @inheritDoc
* {@inheritDoc}
*/
protected function getSpecialTokens(): array
{
Expand Down
4 changes: 2 additions & 2 deletions src/Enums/SpecialToken.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ enum SpecialToken: string
case ENDOFPROMPT = '<|endofprompt|>';

/**
* @param array<string> $tokens
* @return string
* @param array<string> $tokens
*
* @throws InvalidPatternException
*/
public static function getRegex(array $tokens): string
Expand Down
5 changes: 1 addition & 4 deletions src/Loaders/DataGymLoader.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,8 @@
final class DataGymLoader extends Loader
{
/**
* @param string $vocabBpeFile
* @param string $encoderJsonFile
* @param string|null $vocabBpeHash
* @param string|null $encoderJsonHash
* @return non-empty-array<int|string, int>
*
* @throws \Rahul900day\Tiktoken\Exceptions\InvalidChecksumException
*/
public function load(
Expand Down
5 changes: 2 additions & 3 deletions src/Loaders/TiktokenLoader.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@
final class TiktokenLoader extends Loader
{
/**
* @param string $bpeFile
* @param string|null $expectedHash
* @return non-empty-array<int|string, int>
*
* @throws \Rahul900day\Tiktoken\Exceptions\InvalidChecksumException
*/
public function load(string $bpeFile, ?string $expectedHash = null): array
Expand All @@ -26,7 +25,7 @@ public function load(string $bpeFile, ?string $expectedHash = null): array
$result[base64_decode($token)] = intval($rank);
}

if(count($result) === 0) {
if (count($result) === 0) {
throw new \Exception('Invalid tiktoken');
}

Expand Down
4 changes: 2 additions & 2 deletions src/Readers/HttpReader.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ public static function create(?ClientInterface $client = null): HttpReader

public function read(string|RequestInterface $location): string
{
if(is_string($location)) {
if (is_string($location)) {
$request = (new Psr17Factory())->createRequest('GET', $location);
}else {
} else {
$request = $location;
}

Expand Down
4 changes: 2 additions & 2 deletions src/Registry.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class Registry
/** @var array<string, EncodingContract|Closure> */
protected static array $encodings = [];

/** @var non-empty-array<string, array{0: class-string, 1: array}> */
/** @var non-empty-array<string, array{0: class-string, 1: array}> */
public static array $defaultEncodings = [
'gpt2' => [Gpt2Encoding::class, []],
'r50k_base' => [R50KBaseEncoding::class, []],
Expand All @@ -33,7 +33,7 @@ class Registry

protected static function registerEncoding(string $name, EncodingContract|Closure $encoding): void
{
if(isset(self::$resolvedEncodings[$name])) {
if (isset(self::$resolvedEncodings[$name])) {
unset(self::$resolvedEncodings[$name]);
}

Expand Down
15 changes: 6 additions & 9 deletions src/Utils/ArrayUtil.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ final class ArrayUtil
/**
* @template TKey
* @template TValue
* @param array<TKey, TValue> $array
* @param int $at
* @return mixed
*
* @param array<TKey, TValue> $array
*/
public static function &at(array &$array, int $at): mixed
{
Expand All @@ -25,9 +24,8 @@ public static function &at(array &$array, int $at): mixed
/**
* @template TKey
* @template TValue
* @param array<TKey, TValue> $array
* @param int $at
* @return void
*
* @param array<TKey, TValue> $array
*/
public static function unsetAt(array &$array, int $at): void
{
Expand All @@ -39,9 +37,8 @@ public static function unsetAt(array &$array, int $at): void
/**
* @template TKey
* @template TValue
* @param non-empty-array<TKey, TValue> $array
* @param int $start
* @param int $end
*
* @param non-empty-array<TKey, TValue> $array
* @return array<TKey, TValue>
*/
public static function getSegment(array $array, int $start, int $end): array
Expand Down
4 changes: 1 addition & 3 deletions src/Utils/EncoderUtil.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
final class EncoderUtil
{
/**
* @param string $string
* @return array<int>
*/
public static function toBytes(string $string): array
Expand All @@ -16,8 +15,7 @@ public static function toBytes(string $string): array
}

/**
* @param array<int> $bytes
* @return string
* @param array<int> $bytes
*/
public static function fromBytes(array $bytes): string
{
Expand Down
2 changes: 1 addition & 1 deletion src/Vocab.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ final class Vocab implements Countable
public readonly array $rankToTokens;

/**
* @param non-empty-array<string|int, int> $tokenToRanks
* @param non-empty-array<string|int, int> $tokenToRanks
*/
public function __construct(public readonly array $tokenToRanks)
{
Expand Down
Loading

0 comments on commit 1e57e71

Please sign in to comment.