Generate Grammar using PHP Manual as input

.gitignore

@@ -1,2 +1,4 @@
 /Support/generate/.phpdoc
 node_modules
+search-index.json
+tmp

LICENSE.md

@@ -29,3 +29,9 @@ Permission to copy, use, modify, sell and distribute this
 software is granted. This software is provided "as is" without
 express or implied warranty, and with no claim as to its
 suitability for any purpose.
+
+--------------------------------------------------------------------
+
+IMipPlugin.php in spec/fixtures folder is taken from http://sabre.io
+and is licensed from fruux GmbH (https://fruux.com/) under three clause 
+BSD license, see http://sabre.io/license/.

Makefile

@@ -0,0 +1,55 @@
+SHELL = bash
+
+grammars/php.cson: src/php-functions.coffee src/php-constants.coffee
+	coffee src/php.coffee
+
+tmp/php_manual_en.tar.gz:
+	curl -L http://de2.php.net/get/php_manual_en.tar.gz/from/this/mirror -o $@
+
+tmp/php_chunked_xhtml/index.html: tmp/php_manual_en.tar.gz
+	cd tmp && tar xzf ../$<
+
+src/php-functions.coffee: tmp/php_chunked_xhtml/index.html Makefile
+	echo "# do not make changes here, it is generated from php manual" > $@
+	echo "{makeWords} = require 'atom-syntax-tools'" >> $@
+	echo >> $@
+	echo "module.exports =" >> $@
+	CURRENT="" ; \
+	cat tmp/php-chunked-xhtml/function.* | perl -n -e 'if (m{div class="up"><a href="(ref|cubrid.*?|.*?)\.([\w\-]+)\.}) { $$module = $$2 }; m{h1 class="refname">([^<]*)} && print "$$module $$1\n"' \
+	| grep -v "::" | sort | while read MODULE FUNCTION ; do \
+		if [[ "$$MODULE" != "$$CURRENT" ]] ; then \
+			if [[ -n "$$CURRENT" ]] ; then \
+				echo "  \"\"\"" >> $@ ;\
+				echo >> $@ ;\
+			fi ; \
+			echo "  '$$MODULE': makeWords \"\"\"" >> $@ ;\
+			CURRENT=$$MODULE ;\
+		fi ;\
+		echo "    $$FUNCTION" >> $@ ;\
+	done
+	echo "  \"\"\"" >> $@
+
+
+src/php-constants.coffee: tmp/php_chunked_xhtml/index.html Makefile
+	echo "# do not make changes here, it is generated from php manual" > $@
+	echo "{makeWords} = require 'atom-syntax-tools'" >> $@
+	echo >> $@
+	echo "module.exports =" >> $@
+	CURRENT="" ; \
+	grep "^[ ]*<strong><code>" tmp/php-chunked-xhtml/*constants* | perl -n -e 'm{^.*/(.*?)\.constants.*<code>([^<]*?)</code>} && print "$$1 $$2\n"' | sort | while read MODULE FUNCTION ; do \
+		if [[ "$$MODULE" != "$$CURRENT" ]] ; then \
+			if [[ -n "$$CURRENT" ]] ; then \
+				echo "  \"\"\"" >> $@ ;\
+				echo >> $@ ;\
+			fi ; \
+			echo "  '$$MODULE': makeWords \"\"\"" >> $@ ;\
+			CURRENT=$$MODULE ;\
+		fi ;\
+		echo "    $$FUNCTION" >> $@ ;\
+	done
+	echo "  \"\"\"" >> $@
+
+#src/php-builtin-classes.coffee: Makefile
+
+#load json:
+#http://php.net/manual/en/search-index.json

README.md

@@ -7,3 +7,22 @@ from the [PHP TextMate bundle](https://github.com/textmate/php.tmbundle).
 
 Contributions are greatly appreciated. Please fork this repository and open a
 pull request to add snippets, make grammar tweaks, etc.
+
+
+## Development
+
+Grammar `grammars/php.cson` is generated from `src/php.coffee` using PHP Manual as
+input.
+
+For now it is done in an un-nodejs-ish way using `make` having `make`, `bash`,
+`curl` and `perl` involved:
+```
+    make
+```
+
+This will fetch PHP Manual and generate `src/php-constants.coffee`, `src/php-functions.coffee` and deploys the full documentation in
+`tmp/php-chunked-xhtml/`.  File `tmp/php-chunked-xhtml/search-index.json`
+is involved from `src/php-grammar.coffee` to create builtin class lists.
+
+Once having run the initial make, the environment is ready to build php.cson
+each time on running the package tests.

package.json

@@ -16,6 +16,10 @@
     "url": "https://github.com/atom/language-php/issues"
   },
   "devDependencies": {
+    "atom-syntax-tools": "^0.9.1",
+    "q": "^1.4.1",
     "coffeelint": "^1.10.1"
+  },
+  "dependencies": {
   }
 }

spec/fixtures/IMipPlugin.php

@@ -0,0 +1,190 @@
+<?php
+
+namespace Sabre\CalDAV\Schedule;
+
+use Sabre\DAV;
+use Sabre\VObject\ITip;
+
+/**
+ * iMIP handler.
+ *
+ * This class is responsible for sending out iMIP messages. iMIP is the
+ * email-based transport for iTIP. iTIP deals with scheduling operations for
+ * iCalendar objects.
+ *
+ * If you want to customize the email that gets sent out, you can do so by
+ * extending this class and overriding the sendMessage method.
+ *
+ * @copyright Copyright (C) 2007-2015 fruux GmbH (https://fruux.com/).
+ * @author Evert Pot (http://evertpot.com/)
+ * @license http://sabre.io/license/ Modified BSD License
+ */
+class IMipPlugin extends DAV\ServerPlugin {
+
+    /**
+     * Email address used in From: header.
+     *
+     * @var string
+     */
+    protected $senderEmail;
+
+    /**
+     * ITipMessage
+     *
+     * @var ITip\Message
+     */
+    protected $itipMessage;
+
+    /**
+     * Creates the email handler.
+     *
+     * @param string $senderEmail. The 'senderEmail' is the email that shows up
+     *                             in the 'From:' address. This should
+     *                             generally be some kind of no-reply email
+     *                             address you own.
+     */
+    function __construct($senderEmail) {
+
+        $this->senderEmail = $senderEmail;
+
+    }
+
+    /**
+     * This initializes the plugin.
+     *
+     * This function is called by Sabre\DAV\Server, after
+     * addPlugin is called.
+     *
+     * This method should set up the required event subscriptions.
+     *
+     * @param DAV\Server $server
+     * @return void
+     */
+    function initialize(DAV\Server $server) {
+
+        $server->on('schedule', [$this, 'schedule'], 120);
+
+    }
+
+    /**
+     * Returns a plugin name.
+     *
+     * Using this name other plugins will be able to access other plugins
+     * using \Sabre\DAV\Server::getPlugin
+     *
+     * @return string
+     */
+    function getPluginName() {
+
+        return 'imip';
+
+    }
+
+    /**
+     * Event handler for the 'schedule' event.
+     *
+     * @param ITip\Message $iTipMessage
+     * @return void
+     */
+    function schedule(ITip\Message $iTipMessage) {
+
+        // Not sending any emails if the system considers the update
+        // insignificant.
+        if (!$iTipMessage->significantChange) {
+            if (!$iTipMessage->scheduleStatus) {
+                $iTipMessage->scheduleStatus = '1.0;We got the message, but it\'s not significant enough to warrant an email';
+            }
+            return;
+        }
+
+        $summary = $iTipMessage->message->VEVENT->SUMMARY;
+
+        if (parse_url($iTipMessage->sender, PHP_URL_SCHEME) !== 'mailto')
+            return;
+
+        if (parse_url($iTipMessage->recipient, PHP_URL_SCHEME) !== 'mailto')
+            return;
+
+        $sender = substr($iTipMessage->sender, 7);
+        $recipient = substr($iTipMessage->recipient, 7);
+
+        if ($iTipMessage->senderName) {
+            $sender = $iTipMessage->senderName . ' <' . $sender . '>';
+        }
+        if ($iTipMessage->recipientName) {
+            $recipient = $iTipMessage->recipientName . ' <' . $recipient . '>';
+        }
+
+        $subject = 'SabreDAV iTIP message';
+        switch (strtoupper($iTipMessage->method)) {
+            case 'REPLY' :
+                $subject = 'Re: ' . $summary;
+                break;
+            case 'REQUEST' :
+                $subject = $summary;
+                break;
+            case 'CANCEL' :
+                $subject = 'Cancelled: ' . $summary;
+                break;
+        }
+
+        $headers = [
+            'Reply-To: ' . $sender,
+            'From: ' . $this->senderEmail,
+            'Content-Type: text/calendar; charset=UTF-8; method=' . $iTipMessage->method,
+        ];
+        if (DAV\Server::$exposeVersion) {
+            $headers[] = 'X-Sabre-Version: ' . DAV\Version::VERSION;
+        }
+        $this->mail(
+            $recipient,
+            $subject,
+            $iTipMessage->message->serialize(),
+            $headers
+        );
+        $iTipMessage->scheduleStatus = '1.1; Scheduling message is sent via iMip';
+
+    }
+
+    // @codeCoverageIgnoreStart
+    // This is deemed untestable in a reasonable manner
+
+    /**
+     * This function is responsible for sending the actual email.
+     *
+     * @param string $to Recipient email address
+     * @param string $subject Subject of the email
+     * @param string $body iCalendar body
+     * @param array $headers List of headers
+     * @return void
+     */
+    protected function mail($to, $subject, $body, array $headers) {
+
+        mail($to, $subject, $body, implode("\r\n", $headers));
+
+    }
+
+    // @codeCoverageIgnoreEnd
+
+    /**
+     * Returns a bunch of meta-data about the plugin.
+     *
+     * Providing this information is optional, and is mainly displayed by the
+     * Browser plugin.
+     *
+     * The description key in the returned array may contain html and will not
+     * be sanitized.
+     *
+     * @return array
+     */
+    function getPluginInfo() {
+
+        return [
+            'name'        => $this->getPluginName(),
+            'description' => 'Email delivery (rfc6037) for CalDAV scheduling',
+            'link'        => 'http://sabre.io/dav/scheduling/',
+        ];
+
+    }
+
+}

spec/php-spec.coffee

@@ -1,3 +1,5 @@
+#require('../src/php.coffee').createGrammarsCson()
+
 describe 'PHP grammar', ->
   grammar = null
 
@@ -435,3 +437,45 @@ describe 'PHP grammar', ->
         expect(tokens[1][9]).toEqual value: '{', scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php', 'punctuation.section.scope.begin.php']
         expect(tokens[1][10]).toEqual value: '}', scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php', 'punctuation.section.scope.end.php']
         expect(tokens[1][11]).toEqual value: ';', scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php', 'punctuation.terminator.expression.php']
+
+  describe "complex samples", ->
+
+    it 'should tokenize a class definition', ->
+      # this sample is taken from SabreDAV
+      tokens = grammar.tokenizeLines """
+        <?php
+        class IMipPlugin extends DAV\\ServerPlugin {
+
+          /**
+           * Email address used in From: header.
+           *
+           * @var string
+           */
+          protected $senderEmail;
+
+          /**
+           * Creates the email handler.
+           *
+           * @param string $senderEmail. The 'senderEmail' is the email that shows up
+           *                             in the 'From:' address. This should
+           *                             generally be some kind of no-reply email
+           *                             address you own.
+           */
+          function __construct($senderEmail) {
+              $this->senderEmail = $senderEmail;
+          }
+        }
+      """
+
+      expect(tokens[3][1]).toEqual  value: "/**", scopes: ["text.html.php", "meta.embedded.block.php", "source.php", "comment.block.documentation.phpdoc.php", "punctuation.definition.comment.php"]
+      expect(tokens[10][0]).toEqual value: "  ",  scopes: ["text.html.php", "meta.embedded.block.php", "source.php", "comment.block.documentation.phpdoc.php"]
+
+      # TODO: make namespace be lexed in extends
+      #expect(tokens[1][6]).toEqual  value: "DAV",  scopes: ["text.html.php","meta.embedded.block.php","source.php","meta.class.php","meta.other.inherited-class.php","support.other.namespace.php"]
+
+    it 'can tokenize an indented comment', ->
+      path = require 'path'
+      fs = require 'fs'
+      content = fs.readFileSync path.resolve __dirname, "fixtures", "IMipPlugin.php"
+      tokens = grammar.tokenizeLines content.toString()
+      expect(tokens[25][0]).toEqual {value: '     *', scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php', 'comment.block.documentation.phpdoc.php']}